From 35f0ead5ac9a5a10dc0212d8875e18b9bab60b8f Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:55:49 +0800 Subject: [PATCH 01/62] feat: external tls --- src/CraneCtld/CraneCtld.cpp | 37 ++++++++++--------- src/CraneCtld/CranedKeeper.cpp | 5 ++- src/CraneCtld/CtldGrpcServer.cpp | 2 +- src/CraneCtld/CtldPublicDefs.h | 6 ++- src/Utilities/PublicHeader/GrpcHelper.cpp | 4 +- .../PublicHeader/include/crane/GrpcHelper.h | 2 +- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index a159e7626..7c90d1863 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -148,52 +148,53 @@ void ParseConfig(int argc, char** argv) { g_config.CompressedRpc = config["CompressedRpc"].as(); if (config["UseTls"] && config["UseTls"].as()) { - TlsCertificates& tls_certs = g_config.ListenConf.Certs; + TlsCertificates& external_certs = g_config.ListenConf.ExternalCerts; + TlsCertificates& internal_certs = g_config.ListenConf.InternalCerts; g_config.ListenConf.UseTls = true; if (config["DomainSuffix"]) - tls_certs.DomainSuffix = config["DomainSuffix"].as(); + g_config.ListenConf.DomainSuffix = config["DomainSuffix"].as(); - if (config["ServerCertFilePath"]) { - tls_certs.ServerCertFilePath = - config["ServerCertFilePath"].as(); + if (config["CranectldExternalCertFilePath"]) { + external_certs.ServerCertFilePath = + config["CranectldExternalCertFilePath"].as(); try { - tls_certs.ServerCertContent = - util::ReadFileIntoString(tls_certs.ServerCertFilePath); + external_certs.ServerCertContent = + util::ReadFileIntoString(external_certs.ServerCertFilePath); } catch (const std::exception& e) { CRANE_ERROR("Read cert file error: {}", e.what()); std::exit(1); } - if (tls_certs.ServerCertContent.empty()) { + if (external_certs.ServerCertContent.empty()) { CRANE_ERROR( - "UseTls is true, but the file specified by ServerCertFilePath " + "UseTls is true, but the file specified by CranectldExternalCertFilePath " "is empty"); } } else { - CRANE_ERROR("UseTls is true, but ServerCertFilePath is empty"); + CRANE_ERROR("UseTls is true, but CranectldExternalCertFilePath is empty"); std::exit(1); } - if (config["ServerKeyFilePath"]) { - tls_certs.ServerKeyFilePath = - config["ServerKeyFilePath"].as(); + if (config["CranectldExternalKeyFilePath"]) { + external_certs.ServerKeyFilePath = + config["CranectldExternalKeyFilePath"].as(); try { - tls_certs.ServerKeyContent = - util::ReadFileIntoString(tls_certs.ServerKeyFilePath); + external_certs.ServerKeyContent = + util::ReadFileIntoString(external_certs.ServerKeyFilePath); } catch (const std::exception& e) { CRANE_ERROR("Read cert file error: {}", e.what()); std::exit(1); } - if (tls_certs.ServerKeyContent.empty()) { + if (external_certs.ServerKeyContent.empty()) { CRANE_ERROR( - "UseTls is true, but the file specified by ServerKeyFilePath " + "UseTls is true, but the file specified by CranectldExternalKeyFilePath " "is empty"); } } else { - CRANE_ERROR("UseTls is true, but ServerKeyFilePath is empty"); + CRANE_ERROR("UseTls is true, but CranectldExternalKeyFilePath is empty"); std::exit(1); } } else { diff --git a/src/CraneCtld/CranedKeeper.cpp b/src/CraneCtld/CranedKeeper.cpp index bb1c59fb9..b48ee7676 100644 --- a/src/CraneCtld/CranedKeeper.cpp +++ b/src/CraneCtld/CranedKeeper.cpp @@ -726,10 +726,11 @@ void CranedKeeper::ConnectCranedNode_(CranedId const &craned_id) { m_channel_count_.fetch_add(1) + 1); if (g_config.ListenConf.UseTls) { - SetTlsHostnameOverride(&channel_args, craned_id, g_config.ListenConf.Certs); + SetTlsHostnameOverride(&channel_args, craned_id, + g_config.ListenConf.ExternalCerts); craned->m_channel_ = CreateTcpTlsCustomChannelByIp( ip_addr, g_config.CranedListenConf.CranedListenPort, - g_config.ListenConf.Certs, channel_args); + g_config.ListenConf.ExternalCerts, channel_args); } else craned->m_channel_ = CreateTcpInsecureCustomChannel( ip_addr, g_config.CranedListenConf.CranedListenPort, channel_args); diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index f6810cb8c..4bb5b225a 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -899,7 +899,7 @@ CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { if (listen_conf.UseTls) { ServerBuilderAddTcpTlsListeningPort(&builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort, - listen_conf.Certs); + listen_conf.ExternalCerts); } else { ServerBuilderAddTcpInsecureListeningPort(&builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort); diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 72647afe4..1012f7055 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -88,7 +88,11 @@ struct Config { std::string CraneCtldListenPort; bool UseTls{false}; - TlsCertificates Certs; + TlsCertificates ExternalCerts; + TlsCertificates InternalCerts; + TlsCertificates CforedCerts; + std::string InternalCaContent; + std::string DomainSuffix; }; CraneCtldListenConf ListenConf; diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index 4ae2072e4..de4cd73e5 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -77,10 +77,10 @@ void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, // CA certificate. CA certificate is not needed. // Since we use the same cert/key pair for both cranectld/craned, // pem_root_certs is set to the same certificate. - ssl_opts.pem_root_certs = certs.ServerCertContent; + // ssl_opts.pem_root_certs = certs.ServerCertContent; ssl_opts.pem_key_cert_pairs.emplace_back(std::move(pem_key_cert_pair)); ssl_opts.client_certificate_request = - GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY; + GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE; builder->AddListeningPort(listen_addr_port, grpc::SslServerCredentials(ssl_opts)); diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index a4e725aeb..235105865 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -22,11 +22,11 @@ #include struct TlsCertificates { - std::string DomainSuffix; std::string ServerCertFilePath; std::string ServerCertContent; std::string ServerKeyFilePath; std::string ServerKeyContent; + std::string DomainSuffix; }; void ServerBuilderSetCompression(grpc::ServerBuilder* builder); From 3f179e69745c1db7be19bc8b82f78962ff99a575 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:24:50 +0800 Subject: [PATCH 02/62] feat: Separate the CraneCtldForCrane gRPC service and connect Crane to CraneCtld. --- protos/Crane.proto | 48 +++++- src/CraneCtld/CMakeLists.txt | 2 + src/CraneCtld/CraneCtld.cpp | 151 +++++++++++++++++- src/CraneCtld/CranedKeeper.cpp | 9 +- src/CraneCtld/CtldForCranedServer.cpp | 90 +++++++++++ src/CraneCtld/CtldForCranedServer.h | 83 ++++++++++ src/CraneCtld/CtldGrpcServer.cpp | 36 +---- src/CraneCtld/CtldGrpcServer.h | 10 -- src/CraneCtld/CtldPublicDefs.h | 17 +- src/Craned/CforedClient.cpp | 5 +- src/Craned/Craned.cpp | 95 ++++++++--- src/Craned/CranedPublicDefs.h | 11 +- src/Craned/CranedServer.cpp | 21 ++- src/Craned/CtldClient.cpp | 13 +- src/Craned/CtldClient.h | 4 +- src/Utilities/PublicHeader/GrpcHelper.cpp | 57 +++++-- .../PublicHeader/include/crane/GrpcHelper.h | 19 ++- .../PublicHeader/include/crane/PublicHeader.h | 2 + 18 files changed, 555 insertions(+), 118 deletions(-) create mode 100644 src/CraneCtld/CtldForCranedServer.cpp create mode 100644 src/CraneCtld/CtldForCranedServer.h diff --git a/protos/Crane.proto b/protos/Crane.proto index 86dd58b53..4c879a67d 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -749,10 +749,6 @@ message StreamTaskIOReply { // We need to distinguish the message sender // and have some kind of authentication service CraneCtld { - /* RPCs called from Craned */ - rpc TaskStatusChange(TaskStatusChangeRequest) returns (TaskStatusChangeReply); - rpc CranedRegister(CranedRegisterRequest) returns (CranedRegisterReply); - /* RPCs called from Cfored */ rpc CforedStream(stream StreamCforedRequest) returns(stream StreamCtldReply); @@ -795,6 +791,50 @@ service CraneCtld { rpc QueryTasksInfo(QueryTasksInfoRequest) returns (QueryTasksInfoReply); } +service CraneCtldForExternal { + /* RPCs called from ccancel */ + rpc CancelTask(CancelTaskRequest) returns (CancelTaskReply); + + /* RPCs called from cbatch */ + rpc SubmitBatchTask(SubmitBatchTaskRequest) returns (SubmitBatchTaskReply); + rpc SubmitBatchTasks(SubmitBatchTasksRequest) returns (SubmitBatchTasksReply); + + /* PRCs called from ccontrol */ + rpc QueryCranedInfo(QueryCranedInfoRequest) returns (QueryCranedInfoReply); + rpc QueryPartitionInfo(QueryPartitionInfoRequest) returns (QueryPartitionInfoReply); + rpc ModifyTask(ModifyTaskRequest) returns (ModifyTaskReply); + rpc ModifyNode(ModifyCranedStateRequest) returns (ModifyCranedStateReply); + + /* RPCs called from cacctmgr */ + rpc AddAccount(AddAccountRequest) returns (AddAccountReply); + rpc AddUser(AddUserRequest) returns (AddUserReply); + rpc AddQos(AddQosRequest) returns (AddQosReply); + + rpc DeleteEntity(DeleteEntityRequest) returns (DeleteEntityReply); + + rpc QueryEntityInfo(QueryEntityInfoRequest) returns (QueryEntityInfoReply); + rpc ModifyEntity(ModifyEntityRequest) returns (ModifyEntityReply); + rpc BlockAccountOrUser(BlockAccountOrUserRequest) returns (BlockAccountOrUserReply); + + /* RPCs called from cinfo */ + rpc QueryClusterInfo(QueryClusterInfoRequest) returns (QueryClusterInfoReply); + + /* common RPCs */ + rpc QueryTasksInfo(QueryTasksInfoRequest) returns (QueryTasksInfoReply); +} + +service CraneCtldForCraned { + /* RPCs called from Craned */ + rpc TaskStatusChange(TaskStatusChangeRequest) returns (TaskStatusChangeReply); + rpc CranedRegister(CranedRegisterRequest) returns (CranedRegisterReply); +} + +service CraneCtldForCfored { + /* RPCs called from Cfored */ + rpc CforedStream(stream StreamCforedRequest) returns(stream StreamCtldReply); +} + + service Craned { /* ----------------------------------- Called from CraneCtld ---------------------------------------------------- */ rpc ExecuteTask(ExecuteTasksRequest) returns(ExecuteTasksReply); diff --git a/src/CraneCtld/CMakeLists.txt b/src/CraneCtld/CMakeLists.txt index 7ff398223..b5276fa8b 100644 --- a/src/CraneCtld/CMakeLists.txt +++ b/src/CraneCtld/CMakeLists.txt @@ -2,6 +2,8 @@ add_executable(cranectld CtldPublicDefs.h CtldGrpcServer.h CtldGrpcServer.cpp + CtldForCranedServer.h + CtldForCranedServer.cpp DbClient.h DbClient.cpp TaskScheduler.h diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 7c90d1863..9090e917b 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -16,6 +16,7 @@ * along with this program. If not, see . */ +#include "CtldForCranedServer.h" #include "CtldPreCompiledHeader.h" // Precompiled header comes first! @@ -25,6 +26,7 @@ #include #include +#include #include "AccountManager.h" #include "AccountMetaContainer.h" @@ -35,6 +37,8 @@ #include "DbClient.h" #include "EmbeddedDbClient.h" #include "TaskScheduler.h" +#include "crane/GrpcHelper.h" +#include "crane/Logger.h" #include "crane/Network.h" #include "crane/PluginClient.h" @@ -144,17 +148,31 @@ void ParseConfig(int argc, char** argv) { else g_config.ListenConf.CraneCtldListenPort = kCtldDefaultPort; + if (config["CraneCtldForCranedListenPort"]) + g_config.ListenConf.CraneCtldForCranedListenPort = + config["CraneCtldForCranedListenPort"].as(); + else + g_config.ListenConf.CraneCtldForCranedListenPort = + kCtldForCranedDefaultPort; + if (config["CompressedRpc"]) g_config.CompressedRpc = config["CompressedRpc"].as(); if (config["UseTls"] && config["UseTls"].as()) { - TlsCertificates& external_certs = g_config.ListenConf.ExternalCerts; - TlsCertificates& internal_certs = g_config.ListenConf.InternalCerts; + TlsCertificates& external_certs = + g_config.ListenConf.TlsCerts.ExternalCerts; + TlsCertificates& internal_certs = + g_config.ListenConf.TlsCerts.InternalCerts; + ClientTlsCertificates& craned_certs = + g_config.ListenConf.TlsCerts.CranedClientCerts; + ClientTlsCertificates& cfored_certs = + g_config.ListenConf.TlsCerts.CforedClientCerts; g_config.ListenConf.UseTls = true; if (config["DomainSuffix"]) - g_config.ListenConf.DomainSuffix = config["DomainSuffix"].as(); + g_config.ListenConf.TlsCerts.DomainSuffix = + config["DomainSuffix"].as(); if (config["CranectldExternalCertFilePath"]) { external_certs.ServerCertFilePath = @@ -169,11 +187,13 @@ void ParseConfig(int argc, char** argv) { } if (external_certs.ServerCertContent.empty()) { CRANE_ERROR( - "UseTls is true, but the file specified by CranectldExternalCertFilePath " + "UseTls is true, but the file specified by " + "CranectldExternalCertFilePath " "is empty"); } } else { - CRANE_ERROR("UseTls is true, but CranectldExternalCertFilePath is empty"); + CRANE_ERROR( + "UseTls is true, but CranectldExternalCertFilePath is empty"); std::exit(1); } @@ -190,13 +210,125 @@ void ParseConfig(int argc, char** argv) { } if (external_certs.ServerKeyContent.empty()) { CRANE_ERROR( - "UseTls is true, but the file specified by CranectldExternalKeyFilePath " + "UseTls is true, but the file specified by " + "CranectldExternalKeyFilePath " + "is empty"); + } + } else { + CRANE_ERROR( + "UseTls is true, but CranectldExternalKeyFilePath is empty"); + std::exit(1); + } + + if (config["InternalCaFilePath"]) { + std::string internalCaFilePath = + config["InternalCaFilePath"].as(); + + try { + g_config.ListenConf.TlsCerts.InternalCaContent = + util::ReadFileIntoString(internalCaFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (g_config.ListenConf.TlsCerts.InternalCaContent.empty()) { + CRANE_ERROR( + "UseTls is true, but the file specified by InternalCaFilePath " + "is empty"); + } + } else { + CRANE_ERROR("UseTls is true, but InternalCaFilePath is empty"); + std::exit(1); + } + + if (config["CranectldInternalCertFilePath"]) { + internal_certs.ServerCertFilePath = + config["CranectldInternalCertFilePath"].as(); + + try { + internal_certs.ServerCertContent = + util::ReadFileIntoString(internal_certs.ServerCertFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (internal_certs.ServerCertContent.empty()) { + CRANE_ERROR( + "UseTls is true, but the file specified by " + "CranectldInternalCertFilePath " + "is empty"); + } + } else { + CRANE_ERROR( + "UseTls is true, but CranectldInternalCertFilePath is empty"); + std::exit(1); + } + + if (config["CranectldInternalKeyFilePath"]) { + internal_certs.ServerKeyFilePath = + config["CranectldInternalKeyFilePath"].as(); + + try { + internal_certs.ServerKeyContent = + util::ReadFileIntoString(internal_certs.ServerKeyFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (internal_certs.ServerKeyContent.empty()) { + CRANE_ERROR( + "UseTls is true, but the file specified by " + "CranectldInternalKeyFilePath " "is empty"); } } else { - CRANE_ERROR("UseTls is true, but CranectldExternalKeyFilePath is empty"); + CRANE_ERROR( + "UseTls is true, but CranectldInternalKeyFilePath is empty"); std::exit(1); } + + if (config["CranedCertFilePath"]) { + craned_certs.ClientCertFilePath = + config["CranedCertFilePath"].as(); + + try { + craned_certs.ClientCertContent = + util::ReadFileIntoString(craned_certs.ClientCertFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (craned_certs.ClientCertContent.empty()) { + CRANE_ERROR( + "UseTls is true, but the file specified by CranedCertFilePath " + "is empty"); + } + } else { + CRANE_ERROR("UseTls is true, but CranedCertFilePath is empty"); + std::exit(1); + } + + if (config["CforedCertFilePath"]) { + cfored_certs.ClientCertFilePath = + config["CforedCertFilePath"].as(); + + try { + cfored_certs.ClientCertContent = + util::ReadFileIntoString(cfored_certs.ClientCertFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (cfored_certs.ClientCertContent.empty()) { + CRANE_ERROR( + "UseTls is true, but the file specified by CforedCertFilePath " + "is empty"); + } + } else { + CRANE_ERROR("UseTls is true, but CforedCertFilePath is empty"); + std::exit(1); + } + } else { g_config.ListenConf.UseTls = false; } @@ -777,6 +909,9 @@ void InitializeCtldGlobalVariables() { } g_ctld_server = std::make_unique(g_config.ListenConf); + + g_ctld_for_craned_server = + std::make_unique(g_config.ListenConf); } void CreateFolders() { @@ -808,6 +943,8 @@ int StartServer() { g_ctld_server->Wait(); + g_ctld_for_craned_server->Wait(); + DestroyCtldGlobalVariables(); return 0; diff --git a/src/CraneCtld/CranedKeeper.cpp b/src/CraneCtld/CranedKeeper.cpp index b48ee7676..9160ba741 100644 --- a/src/CraneCtld/CranedKeeper.cpp +++ b/src/CraneCtld/CranedKeeper.cpp @@ -18,6 +18,10 @@ #include "CranedKeeper.h" +#include + +#include "CtldPublicDefs.h" + namespace Ctld { using grpc::ClientContext; @@ -727,10 +731,11 @@ void CranedKeeper::ConnectCranedNode_(CranedId const &craned_id) { if (g_config.ListenConf.UseTls) { SetTlsHostnameOverride(&channel_args, craned_id, - g_config.ListenConf.ExternalCerts); + g_config.ListenConf.TlsCerts.DomainSuffix); craned->m_channel_ = CreateTcpTlsCustomChannelByIp( ip_addr, g_config.CranedListenConf.CranedListenPort, - g_config.ListenConf.ExternalCerts, channel_args); + g_config.ListenConf.TlsCerts.InternalCerts, + g_config.ListenConf.TlsCerts.CranedClientCerts, channel_args); } else craned->m_channel_ = CreateTcpInsecureCustomChannel( ip_addr, g_config.CranedListenConf.CranedListenPort, channel_args); diff --git a/src/CraneCtld/CtldForCranedServer.cpp b/src/CraneCtld/CtldForCranedServer.cpp new file mode 100644 index 000000000..be25e7f52 --- /dev/null +++ b/src/CraneCtld/CtldForCranedServer.cpp @@ -0,0 +1,90 @@ +/** + * Copyright (c) 2023 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * CraneSched is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of + * the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, + * WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#include "CtldForCranedServer.h" + +#include + +#include "CranedKeeper.h" +#include "CranedMetaContainer.h" +#include "TaskScheduler.h" +#include "crane/String.h" + +namespace Ctld { + +grpc::Status CtldForCranedServiceImpl::TaskStatusChange( + grpc::ServerContext *context, + const crane::grpc::TaskStatusChangeRequest *request, + crane::grpc::TaskStatusChangeReply *response) { + std::optional reason; + if (!request->reason().empty()) reason = request->reason(); + + g_task_scheduler->TaskStatusChangeWithReasonAsync( + request->task_id(), request->craned_id(), request->new_status(), + request->exit_code(), std::move(reason)); + response->set_ok(true); + return grpc::Status::OK; +} + +grpc::Status CtldForCranedServiceImpl::CranedRegister( + grpc::ServerContext *context, + const crane::grpc::CranedRegisterRequest *request, + crane::grpc::CranedRegisterReply *response) { + if (!g_meta_container->CheckCranedAllowed(request->craned_id())) { + response->set_ok(false); + return grpc::Status::OK; + } + + bool alive = g_meta_container->CheckCranedOnline(request->craned_id()); + if (!alive) { + g_craned_keeper->PutNodeIntoUnavailList(request->craned_id()); + } + + response->set_ok(true); + response->set_already_registered(alive); + + return grpc::Status::OK; +} + +CtldForCranedServer::CtldForCranedServer(const Config::CraneCtldListenConf &listen_conf) { + m_service_impl_ = std::make_unique(this); + + grpc::ServerBuilder builder; + + if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); + + if (listen_conf.UseTls) { + ServerBuilderAddmTcpTlsListeningPort( + &builder, listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForCranedListenPort, listen_conf.TlsCerts.InternalCerts, listen_conf.TlsCerts.InternalCaContent); + } else { + ServerBuilderAddTcpInsecureListeningPort(&builder, + listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForCranedListenPort); + } + + builder.RegisterService(m_service_impl_.get()); + m_server_ = builder.BuildAndStart(); + if (!m_server_) { + CRANE_ERROR("Cannot start gRPC server!"); + std::exit(1); + } + CRANE_INFO("CraneCtld For Craned Server is listening on {}:{} and Tls is {}", + listen_conf.CraneCtldListenAddr, listen_conf.CraneCtldForCranedListenPort, + listen_conf.UseTls); +} + +} // namespace Ctld \ No newline at end of file diff --git a/src/CraneCtld/CtldForCranedServer.h b/src/CraneCtld/CtldForCranedServer.h new file mode 100644 index 000000000..c29af8fbb --- /dev/null +++ b/src/CraneCtld/CtldForCranedServer.h @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2023 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * CraneSched is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of + * the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, + * WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + + #pragma once + +#include "CtldPublicDefs.h" +// Precompiled header comes first! + +#include "crane/Lock.h" +#include "protos/Crane.grpc.pb.h" +#include "protos/Crane.pb.h" + +namespace Ctld { + +using crane::grpc::Craned; +using grpc::Channel; +using grpc::Server; + +class CtldForCranedServer; + +class CtldForCranedServiceImpl final : public crane::grpc::CraneCtldForCraned::Service { + public: + explicit CtldForCranedServiceImpl(CtldForCranedServer* server) : m_ctld_for_craned_server_(server) {} + grpc::Status TaskStatusChange( + grpc::ServerContext *context, + const crane::grpc::TaskStatusChangeRequest *request, + crane::grpc::TaskStatusChangeReply *response) override; + + grpc::Status CranedRegister( + grpc::ServerContext *context, + const crane::grpc::CranedRegisterRequest *request, + crane::grpc::CranedRegisterReply *response) override; + private: + CtldForCranedServer *m_ctld_for_craned_server_; +}; + +class CtldForCranedServer { + public: + /*** + * User must make sure that this constructor is called only once! + * @param listen_address The "[Address]:[Port]" of CraneCtld. + */ + explicit CtldForCranedServer(const Config::CraneCtldListenConf &listen_conf); + + inline void Wait() { m_server_->Wait(); } + + private: + template > + using HashMap = absl::flat_hash_map; + + template > + using HashSet = absl::flat_hash_set; + + using Mutex = util::mutex; + + std::unique_ptr m_service_impl_; + std::unique_ptr m_server_; + + inline static std::mutex s_sigint_mtx; + inline static std::condition_variable s_sigint_cv; + static void signal_handler_func(int) { s_sigint_cv.notify_one(); }; + + friend class CtldForCranedServiceImpl; +}; + +} // namespace Ctld + +inline std::unique_ptr g_ctld_for_craned_server; \ No newline at end of file diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index 4bb5b225a..8904d10a8 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -81,40 +81,6 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTasks( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::TaskStatusChange( - grpc::ServerContext *context, - const crane::grpc::TaskStatusChangeRequest *request, - crane::grpc::TaskStatusChangeReply *response) { - std::optional reason; - if (!request->reason().empty()) reason = request->reason(); - - g_task_scheduler->TaskStatusChangeWithReasonAsync( - request->task_id(), request->craned_id(), request->new_status(), - request->exit_code(), std::move(reason)); - response->set_ok(true); - return grpc::Status::OK; -} - -grpc::Status CraneCtldServiceImpl::CranedRegister( - grpc::ServerContext *context, - const crane::grpc::CranedRegisterRequest *request, - crane::grpc::CranedRegisterReply *response) { - if (!g_meta_container->CheckCranedAllowed(request->craned_id())) { - response->set_ok(false); - return grpc::Status::OK; - } - - bool alive = g_meta_container->CheckCranedOnline(request->craned_id()); - if (!alive) { - g_craned_keeper->PutNodeIntoUnavailList(request->craned_id()); - } - - response->set_ok(true); - response->set_already_registered(alive); - - return grpc::Status::OK; -} - grpc::Status CraneCtldServiceImpl::CancelTask( grpc::ServerContext *context, const crane::grpc::CancelTaskRequest *request, crane::grpc::CancelTaskReply *response) { @@ -899,7 +865,7 @@ CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { if (listen_conf.UseTls) { ServerBuilderAddTcpTlsListeningPort(&builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort, - listen_conf.ExternalCerts); + listen_conf.TlsCerts.ExternalCerts); } else { ServerBuilderAddTcpInsecureListeningPort(&builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort); diff --git a/src/CraneCtld/CtldGrpcServer.h b/src/CraneCtld/CtldGrpcServer.h index d35493fec..59255b53a 100644 --- a/src/CraneCtld/CtldGrpcServer.h +++ b/src/CraneCtld/CtldGrpcServer.h @@ -189,16 +189,6 @@ class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { const crane::grpc::SubmitBatchTasksRequest *request, crane::grpc::SubmitBatchTasksReply *response) override; - grpc::Status TaskStatusChange( - grpc::ServerContext *context, - const crane::grpc::TaskStatusChangeRequest *request, - crane::grpc::TaskStatusChangeReply *response) override; - - grpc::Status CranedRegister( - grpc::ServerContext *context, - const crane::grpc::CranedRegisterRequest *request, - crane::grpc::CranedRegisterReply *response) override; - grpc::Status CancelTask(grpc::ServerContext *context, const crane::grpc::CancelTaskRequest *request, crane::grpc::CancelTaskReply *response) override; diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 1012f7055..3bfbc053e 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -19,6 +19,7 @@ #pragma once #include "CtldPreCompiledHeader.h" +#include "crane/GrpcHelper.h" // Precompiled header come first! namespace Ctld { @@ -86,13 +87,19 @@ struct Config { struct CraneCtldListenConf { std::string CraneCtldListenAddr; std::string CraneCtldListenPort; + std::string CraneCtldForCranedListenPort; bool UseTls{false}; - TlsCertificates ExternalCerts; - TlsCertificates InternalCerts; - TlsCertificates CforedCerts; - std::string InternalCaContent; - std::string DomainSuffix; + struct TlsCertsConfig { + std::string InternalCaContent; + std::string DomainSuffix; + TlsCertificates ExternalCerts; + TlsCertificates InternalCerts; + ClientTlsCertificates CranedClientCerts; + ClientTlsCertificates CforedClientCerts; + }; + + TlsCertsConfig TlsCerts; }; CraneCtldListenConf ListenConf; diff --git a/src/Craned/CforedClient.cpp b/src/Craned/CforedClient.cpp index 08a23a13e..8a3649a4b 100644 --- a/src/Craned/CforedClient.cpp +++ b/src/Craned/CforedClient.cpp @@ -44,8 +44,9 @@ void CforedClient::InitChannelAndStub(const std::string& cfored_name) { // Todo: Use cfored listen config if (g_config.ListenConf.UseTls) { - m_cfored_channel_ = CreateTcpTlsChannelByHostname( - cfored_name, kCforedDefaultPort, g_config.ListenConf.TlsCerts); + // m_cfored_channel_ = CreateTcpTlsChannelByHostname( + // cfored_name, kCforedDefaultPort, g_config.ListenConf.TlsCerts); + CreateTcpInsecureChannel(cfored_name, kCforedDefaultPort); } else { m_cfored_channel_ = CreateTcpInsecureChannel(cfored_name, kCforedDefaultPort); diff --git a/src/Craned/Craned.cpp b/src/Craned/Craned.cpp index 437497fa8..dadf8cfe8 100644 --- a/src/Craned/Craned.cpp +++ b/src/Craned/Craned.cpp @@ -25,11 +25,14 @@ #include #include +#include #include "CforedClient.h" #include "CranedServer.h" #include "CtldClient.h" -#include "DeviceManager.h" +#include "crane/GrpcHelper.h" +#include "crane/Network.h" +#include "crane/OS.h" #include "crane/PluginClient.h" #include "crane/String.h" @@ -171,50 +174,98 @@ void ParseConfig(int argc, char** argv) { if (config["UseTls"] && config["UseTls"].as()) { g_config.ListenConf.UseTls = true; - TlsCertificates& tls_certs = g_config.ListenConf.TlsCerts; + TlsCertificates& craned_certs = + g_config.ListenConf.TlsCerts.CranedTlsCerts; + ClientTlsCertificates& internal_client_certs = + g_config.ListenConf.TlsCerts.InternalClientTlsCerts; if (config["DomainSuffix"]) - tls_certs.DomainSuffix = config["DomainSuffix"].as(); + g_config.ListenConf.TlsCerts.DomainSuffix = + config["DomainSuffix"].as(); - if (config["ServerCertFilePath"]) { - tls_certs.ServerCertFilePath = - config["ServerCertFilePath"].as(); + if (config["InternalCaFilePath"]) { + std::string internalCaFilePath = + config["InternalCaFilePath"].as(); try { - tls_certs.ServerCertContent = - util::ReadFileIntoString(tls_certs.ServerCertFilePath); + g_config.ListenConf.TlsCerts.InternalCaContent = + util::ReadFileIntoString(internalCaFilePath); } catch (const std::exception& e) { CRANE_ERROR("Read cert file error: {}", e.what()); std::exit(1); } - if (tls_certs.ServerCertContent.empty()) { + if (g_config.ListenConf.TlsCerts.InternalCaContent.empty()) { CRANE_ERROR( - "UseTls is true, but the file specified by ServerCertFilePath " + "UseTls is true, but the file specified by InternalCaFilePath " "is empty"); } } else { - CRANE_ERROR("UseTls is true, but ServerCertFilePath is empty"); + CRANE_ERROR("UseTls is true, but InternalCaFilePath is empty"); std::exit(1); } - if (config["ServerKeyFilePath"]) { - tls_certs.ServerKeyFilePath = - config["ServerKeyFilePath"].as(); + if (config["CranedCertFilePath"]) { + craned_certs.ServerCertFilePath = + config["CranedCertFilePath"].as(); try { - tls_certs.ServerKeyContent = - util::ReadFileIntoString(tls_certs.ServerKeyFilePath); + craned_certs.ServerCertContent = + util::ReadFileIntoString(craned_certs.ServerCertFilePath); } catch (const std::exception& e) { CRANE_ERROR("Read cert file error: {}", e.what()); std::exit(1); } - if (tls_certs.ServerKeyContent.empty()) { + if (craned_certs.ServerCertContent.empty()) { CRANE_ERROR( - "UseTls is true, but the file specified by ServerKeyFilePath " + "UseTls is true, but the file specified by CranedCertFilePath " "is empty"); } } else { - CRANE_ERROR("UseTls is true, but ServerKeyFilePath is empty"); + CRANE_ERROR("UseTls is true, but CranedCertFilePath is empty"); + std::exit(1); + } + + if (config["CranedKeyFilePath"]) { + craned_certs.ServerKeyFilePath = + config["CranedKeyFilePath"].as(); + + try { + craned_certs.ServerKeyContent = + util::ReadFileIntoString(craned_certs.ServerKeyFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (craned_certs.ServerKeyContent.empty()) { + CRANE_ERROR( + "UseTls is true, but the file specified by CranedKeyFilePath " + "is empty"); + } + } else { + CRANE_ERROR("UseTls is true, but CranedKeyFilePath is empty"); + std::exit(1); + } + + if (config["CranectldInternalCertFilePath"]) { + internal_client_certs.ClientCertFilePath = + config["CranectldInternalCertFilePath"].as(); + + try { + internal_client_certs.ClientCertContent = util::ReadFileIntoString( + internal_client_certs.ClientCertFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (internal_client_certs.ClientCertContent.empty()) { + CRANE_ERROR( + "UseTls is true, but the file specified by " + "CranectldInternalCertFilePath " + "is empty"); + } + } else { + CRANE_ERROR( + "UseTls is true, but CranectldInternalCertFilePath is empty"); std::exit(1); } } else { @@ -231,6 +282,12 @@ void ParseConfig(int argc, char** argv) { else g_config.CraneCtldListenPort = kCtldDefaultPort; + if (config["CraneCtldForCranedListenPort"]) + g_config.CraneCtldForCranedPort = + config["CraneCtldForCranedListenPort"].as(); + else + g_config.CraneCtldForCranedPort = kCtldForCranedDefaultPort; + if (config["Nodes"]) { for (auto it = config["Nodes"].begin(); it != config["Nodes"].end(); ++it) { diff --git a/src/Craned/CranedPublicDefs.h b/src/Craned/CranedPublicDefs.h index 3674aff39..5bd3c1620 100644 --- a/src/Craned/CranedPublicDefs.h +++ b/src/Craned/CranedPublicDefs.h @@ -21,6 +21,7 @@ #include "CranedPreCompiledHeader.h" // Precompiled header comes first +#include "crane/GrpcHelper.h" #include "crane/OS.h" namespace Craned { @@ -54,7 +55,14 @@ struct Config { std::string CranedListenPort; bool UseTls{false}; - TlsCertificates TlsCerts; + struct TlsCertsConfig { + std::string InternalCaContent; + std::string DomainSuffix; + TlsCertificates CranedTlsCerts; + ClientTlsCertificates InternalClientTlsCerts; + }; + + TlsCertsConfig TlsCerts; std::string UnixSocketListenAddr; }; @@ -70,6 +78,7 @@ struct Config { std::string ControlMachine; std::string CraneCtldListenPort; + std::string CraneCtldForCranedPort; std::string CranedDebugLevel; std::string CraneBaseDir; diff --git a/src/Craned/CranedServer.cpp b/src/Craned/CranedServer.cpp index 2b3406c6d..a9785defb 100644 --- a/src/Craned/CranedServer.cpp +++ b/src/Craned/CranedServer.cpp @@ -20,7 +20,8 @@ #include -#include "TaskManager.h" +#include "CranedPublicDefs.h" +#include "CtldClient.h" namespace Craned { @@ -107,7 +108,7 @@ grpc::Status CranedServiceImpl::QueryTaskIdFromPort( } for (auto const &fd_dir_entry : std::filesystem::directory_iterator(proc_fd_path)) { - struct stat statbuf {}; + struct stat statbuf{}; std::string fdpath = fmt::format( "{}/{}", proc_fd_path, fd_dir_entry.path().filename().string()); const char *fdchar = fdpath.c_str(); @@ -288,7 +289,10 @@ grpc::Status CranedServiceImpl::QueryTaskIdFromPortForward( request->ssh_remote_address(), remote_hostname); channel_of_remote_service = CreateTcpTlsChannelByHostname( - remote_hostname, crane_port, g_config.ListenConf.TlsCerts); + remote_hostname, crane_port, + g_config.ListenConf.TlsCerts.CranedTlsCerts, + g_config.ListenConf.TlsCerts.InternalClientTlsCerts, + g_config.ListenConf.TlsCerts.DomainSuffix); } else { CRANE_ERROR("Failed to resolve remote address {}.", request->ssh_remote_address()); @@ -429,7 +433,9 @@ grpc::Status CranedServiceImpl::QueryTaskEnvVariablesForward( if (g_config.ListenConf.UseTls) channel_of_remote_service = CreateTcpTlsChannelByHostname( execution_node, g_config.ListenConf.CranedListenPort, - g_config.ListenConf.TlsCerts); + g_config.ListenConf.TlsCerts.CranedTlsCerts, + g_config.ListenConf.TlsCerts.InternalClientTlsCerts, + g_config.ListenConf.TlsCerts.DomainSuffix); else channel_of_remote_service = CreateTcpInsecureChannel( execution_node, g_config.ListenConf.CranedListenPort); @@ -530,9 +536,10 @@ CranedServer::CranedServer(const Config::CranedListenConf &listen_conf) { std::string craned_listen_addr = listen_conf.CranedListenAddr; if (listen_conf.UseTls) { - ServerBuilderAddTcpTlsListeningPort(&builder, craned_listen_addr, - listen_conf.CranedListenPort, - listen_conf.TlsCerts); + ServerBuilderAddmTcpTlsListeningPort( + &builder, craned_listen_addr, listen_conf.CranedListenPort, + listen_conf.TlsCerts.CranedTlsCerts, + listen_conf.TlsCerts.InternalCaContent); } else { ServerBuilderAddTcpInsecureListeningPort(&builder, craned_listen_addr, listen_conf.CranedListenPort); diff --git a/src/Craned/CtldClient.cpp b/src/Craned/CtldClient.cpp index fbdd1892e..06aec4df2 100644 --- a/src/Craned/CtldClient.cpp +++ b/src/Craned/CtldClient.cpp @@ -17,6 +17,8 @@ */ #include "CtldClient.h" +#include "CranedPublicDefs.h" +#include "crane/GrpcHelper.h" namespace Craned { @@ -33,16 +35,17 @@ void CtldClient::InitChannelAndStub(const std::string& server_address) { if (g_config.CompressedRpc) channel_args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP); - if (g_config.ListenConf.UseTls) + if (g_config.ListenConf.UseTls) m_ctld_channel_ = CreateTcpTlsCustomChannelByHostname( - server_address, g_config.CraneCtldListenPort, - g_config.ListenConf.TlsCerts, channel_args); + server_address, g_config.CraneCtldForCranedPort, + g_config.ListenConf.TlsCerts.CranedTlsCerts, g_config.ListenConf.TlsCerts.InternalClientTlsCerts, + g_config.ListenConf.TlsCerts.DomainSuffix, channel_args); else m_ctld_channel_ = CreateTcpInsecureCustomChannel( - server_address, g_config.CraneCtldListenPort, channel_args); + server_address, g_config.CraneCtldForCranedPort, channel_args); // std::unique_ptr will automatically release the dangling stub. - m_stub_ = CraneCtld::NewStub(m_ctld_channel_); + m_stub_ = CraneCtldForCraned::NewStub(m_ctld_channel_); m_async_send_thread_ = std::thread([this] { AsyncSendThread_(); }); } diff --git a/src/Craned/CtldClient.h b/src/Craned/CtldClient.h index 338850477..9567974da 100644 --- a/src/Craned/CtldClient.h +++ b/src/Craned/CtldClient.h @@ -25,7 +25,7 @@ namespace Craned { -using crane::grpc::CraneCtld; +using crane::grpc::CraneCtldForCraned; using crane::grpc::CranedRegisterReply; using crane::grpc::CranedRegisterRequest; using grpc::Channel; @@ -76,7 +76,7 @@ class CtldClient { std::shared_ptr m_ctld_channel_; - std::unique_ptr m_stub_; + std::unique_ptr m_stub_; CranedId m_craned_id_; diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index de4cd73e5..c95521101 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -86,6 +86,32 @@ void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, grpc::SslServerCredentials(ssl_opts)); } +void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, + const std::string& address, + const std::string& port, + const TlsCertificates& certs, + const std::string pem_root_cert) { + std::string listen_addr_port = + fmt::format("{}:{}", GrpcFormatIpAddress(address), port); + + grpc::SslServerCredentialsOptions::PemKeyCertPair pem_key_cert_pair; + pem_key_cert_pair.cert_chain = certs.ServerCertContent; + pem_key_cert_pair.private_key = certs.ServerKeyContent; + + grpc::SslServerCredentialsOptions ssl_opts; + // pem_root_certs is actually the certificate of server side rather than + // CA certificate. CA certificate is not needed. + // Since we use the same cert/key pair for both cranectld/craned, + // pem_root_certs is set to the same certificate. + ssl_opts.pem_root_certs = pem_root_cert; + ssl_opts.pem_key_cert_pairs.emplace_back(std::move(pem_key_cert_pair)); + ssl_opts.client_certificate_request = + GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY; + + builder->AddListeningPort(listen_addr_port, + grpc::SslServerCredentials(ssl_opts)); +} + void SetGrpcClientKeepAliveChannelArgs(grpc::ChannelArguments* args) { args->SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, 1000 /*ms*/); args->SetInt(GRPC_ARG_MIN_RECONNECT_BACKOFF_MS, 2 /*s*/ * 1000 @@ -105,9 +131,8 @@ void SetGrpcClientKeepAliveChannelArgs(grpc::ChannelArguments* args) { void SetTlsHostnameOverride(grpc::ChannelArguments* args, const std::string& hostname, - const TlsCertificates& certs) { - args->SetSslTargetNameOverride( - fmt::format("{}.{}", hostname, certs.DomainSuffix)); + const std::string& domainSuffix) { + args->SetSslTargetNameOverride(fmt::format("{}.{}", hostname, domainSuffix)); } std::shared_ptr CreateUnixInsecureChannel( @@ -130,21 +155,23 @@ std::shared_ptr CreateTcpInsecureCustomChannel( } static void SetSslCredOpts(grpc::SslCredentialsOptions* opts, - const TlsCertificates& certs) { + const TlsCertificates& certs, + const ClientTlsCertificates& clientcerts) { // pem_root_certs is actually the certificate of server side rather than // CA certificate. CA certificate is not needed. // Since we use the same cert/key pair for both cranectld/craned, // pem_root_certs is set to the same certificate. - opts->pem_root_certs = certs.ServerCertContent; + opts->pem_root_certs = clientcerts.ClientCertContent; opts->pem_cert_chain = certs.ServerCertContent; opts->pem_private_key = certs.ServerKeyContent; } std::shared_ptr CreateTcpTlsCustomChannelByIp( const std::string& ip, const std::string& port, - const TlsCertificates& certs, const grpc::ChannelArguments& args) { + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const grpc::ChannelArguments& args) { grpc::SslCredentialsOptions ssl_opts; - SetSslCredOpts(&ssl_opts, certs); + SetSslCredOpts(&ssl_opts, certs, clientcerts); std::string target = fmt::format("{}:{}", GrpcFormatIpAddress(ip), port); return grpc::CreateCustomChannel(target, grpc::SslCredentials(ssl_opts), @@ -153,23 +180,23 @@ std::shared_ptr CreateTcpTlsCustomChannelByIp( std::shared_ptr CreateTcpTlsChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs) { + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const std::string& domainSuffix) { grpc::SslCredentialsOptions ssl_opts; - SetSslCredOpts(&ssl_opts, certs); + SetSslCredOpts(&ssl_opts, certs, clientcerts); - std::string target = - fmt::format("{}.{}:{}", hostname, certs.DomainSuffix, port); + std::string target = fmt::format("{}.{}:{}", hostname, domainSuffix, port); return grpc::CreateChannel(target, grpc::SslCredentials(ssl_opts)); } std::shared_ptr CreateTcpTlsCustomChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs, const grpc::ChannelArguments& args) { + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const std::string& domainSuffix, const grpc::ChannelArguments& args) { grpc::SslCredentialsOptions ssl_opts; - SetSslCredOpts(&ssl_opts, certs); + SetSslCredOpts(&ssl_opts, certs, clientcerts); - std::string target = - fmt::format("{}.{}:{}", hostname, certs.DomainSuffix, port); + std::string target = fmt::format("{}.{}:{}", hostname, domainSuffix, port); return grpc::CreateCustomChannel(target, grpc::SslCredentials(ssl_opts), args); } diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index 235105865..f70653300 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -29,6 +29,12 @@ struct TlsCertificates { std::string DomainSuffix; }; +struct ClientTlsCertificates { + std::string ClientCertFilePath; + std::string ClientCertContent; +}; + + void ServerBuilderSetCompression(grpc::ServerBuilder* builder); void ServerBuilderSetKeepAliveArgs(grpc::ServerBuilder* builder); @@ -40,6 +46,11 @@ void ServerBuilderAddTcpInsecureListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port); +void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, + const std::string& address, + const std::string& port, + const TlsCertificates& certs, const std::string pem_root_cert); + void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, @@ -49,7 +60,7 @@ void SetGrpcClientKeepAliveChannelArgs(grpc::ChannelArguments* args); void SetTlsHostnameOverride(grpc::ChannelArguments* args, const std::string& hostname, - const TlsCertificates& certs); + const std::string& domainSuffix); std::shared_ptr CreateUnixInsecureChannel( const std::string& socket_addr); @@ -63,12 +74,12 @@ std::shared_ptr CreateTcpInsecureCustomChannel( std::shared_ptr CreateTcpTlsCustomChannelByIp( const std::string& ip, const std::string& port, - const TlsCertificates& certs, const grpc::ChannelArguments& args); + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, const grpc::ChannelArguments& args); std::shared_ptr CreateTcpTlsChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs); + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, const std::string& domainSuffix); std::shared_ptr CreateTcpTlsCustomChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs, const grpc::ChannelArguments& args); + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, const std::string& domainSuffix, const grpc::ChannelArguments& args); diff --git a/src/Utilities/PublicHeader/include/crane/PublicHeader.h b/src/Utilities/PublicHeader/include/crane/PublicHeader.h index fbb080ada..cdd95342d 100644 --- a/src/Utilities/PublicHeader/include/crane/PublicHeader.h +++ b/src/Utilities/PublicHeader/include/crane/PublicHeader.h @@ -67,6 +67,8 @@ using CraneExpected = std::expected; inline const char* kCtldDefaultPort = "10011"; inline const char* kCranedDefaultPort = "10010"; inline const char* kCforedDefaultPort = "10012"; +inline const char* kCtldForCranedDefaultPort = "10013"; +inline const char* kCtldForCforedDefaultPort = "10014"; inline const char* kDefaultConfigPath = "/etc/crane/config.yaml"; inline const char* kDefaultDbConfigPath = "/etc/crane/database.yaml"; From 8a4fbd58d2221a8e7eca0203cd4f5c964ac1404f Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:40:30 +0800 Subject: [PATCH 03/62] feat: Separate the CraneCtldForCfored gRPC service and connect Cfored to CraneCtld. --- protos/Crane.proto | 43 ---- src/CraneCtld/CMakeLists.txt | 2 + src/CraneCtld/CraneCtld.cpp | 21 ++ src/CraneCtld/CranedKeeper.cpp | 13 +- src/CraneCtld/CtldForCforedServer.cpp | 324 ++++++++++++++++++++++++++ src/CraneCtld/CtldForCforedServer.h | 215 +++++++++++++++++ src/CraneCtld/CtldForCranedServer.cpp | 11 +- src/CraneCtld/CtldGrpcServer.cpp | 179 -------------- src/CraneCtld/CtldGrpcServer.h | 155 ------------ src/CraneCtld/CtldPublicDefs.h | 1 + src/Craned/CforedClient.cpp | 14 +- src/Craned/Craned.cpp | 25 ++ src/Craned/CranedPublicDefs.h | 1 + 13 files changed, 608 insertions(+), 396 deletions(-) create mode 100644 src/CraneCtld/CtldForCforedServer.cpp create mode 100644 src/CraneCtld/CtldForCforedServer.h diff --git a/protos/Crane.proto b/protos/Crane.proto index 4c879a67d..0a1199e84 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -749,49 +749,6 @@ message StreamTaskIOReply { // We need to distinguish the message sender // and have some kind of authentication service CraneCtld { - /* RPCs called from Cfored */ - rpc CforedStream(stream StreamCforedRequest) returns(stream StreamCtldReply); - - /* RPCs called from ccancel */ - rpc CancelTask(CancelTaskRequest) returns (CancelTaskReply); - - /* RPCs called from cbatch */ - rpc SubmitBatchTask(SubmitBatchTaskRequest) returns (SubmitBatchTaskReply); - rpc SubmitBatchTasks(SubmitBatchTasksRequest) returns (SubmitBatchTasksReply); - - /* PRCs called from ccontrol */ - rpc QueryCranedInfo(QueryCranedInfoRequest) returns (QueryCranedInfoReply); - rpc QueryPartitionInfo(QueryPartitionInfoRequest) returns (QueryPartitionInfoReply); - rpc ModifyTask(ModifyTaskRequest) returns (ModifyTaskReply); - rpc ModifyNode(ModifyCranedStateRequest) returns (ModifyCranedStateReply); - - /* RPCs called from cacctmgr */ - rpc AddAccount(AddAccountRequest) returns (AddAccountReply); - rpc AddUser(AddUserRequest) returns (AddUserReply); - rpc AddQos(AddQosRequest) returns (AddQosReply); - - rpc DeleteAccount(DeleteAccountRequest) returns (DeleteAccountReply); - rpc DeleteUser(DeleteUserRequest) returns (DeleteUserReply); - rpc DeleteQos(DeleteQosRequest) returns (DeleteQosReply); - - rpc QueryAccountInfo(QueryAccountInfoRequest) returns (QueryAccountInfoReply); - rpc QueryUserInfo(QueryUserInfoRequest) returns (QueryUserInfoReply); - rpc QueryQosInfo(QueryQosInfoRequest) returns (QueryQosInfoReply); - - rpc ModifyAccount(ModifyAccountRequest) returns (ModifyAccountReply); - rpc ModifyUser(ModifyUserRequest) returns (ModifyUserReply); - rpc ModifyQos(ModifyQosRequest) returns (ModifyQosReply); - - rpc BlockAccountOrUser(BlockAccountOrUserRequest) returns (BlockAccountOrUserReply); - - /* RPCs called from cinfo */ - rpc QueryClusterInfo(QueryClusterInfoRequest) returns (QueryClusterInfoReply); - - /* common RPCs */ - rpc QueryTasksInfo(QueryTasksInfoRequest) returns (QueryTasksInfoReply); -} - -service CraneCtldForExternal { /* RPCs called from ccancel */ rpc CancelTask(CancelTaskRequest) returns (CancelTaskReply); diff --git a/src/CraneCtld/CMakeLists.txt b/src/CraneCtld/CMakeLists.txt index b5276fa8b..f0caaca12 100644 --- a/src/CraneCtld/CMakeLists.txt +++ b/src/CraneCtld/CMakeLists.txt @@ -4,6 +4,8 @@ add_executable(cranectld CtldGrpcServer.cpp CtldForCranedServer.h CtldForCranedServer.cpp + CtldForCforedServer.h + CtldForCforedServer.cpp DbClient.h DbClient.cpp TaskScheduler.h diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 9090e917b..f0ed3d4ba 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -16,6 +16,7 @@ * along with this program. If not, see . */ +#include "CtldForCforedServer.h" #include "CtldForCranedServer.h" #include "CtldPreCompiledHeader.h" // Precompiled header comes first! @@ -41,6 +42,7 @@ #include "crane/Logger.h" #include "crane/Network.h" #include "crane/PluginClient.h" +#include "crane/PublicHeader.h" void ParseConfig(int argc, char** argv) { cxxopts::Options options("cranectld"); @@ -155,6 +157,20 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.CraneCtldForCranedListenPort = kCtldForCranedDefaultPort; + if (config["CraneCtldForCforedListenPort"]) + g_config.ListenConf.CraneCtldForCforedListenPort = + config["CraneCtldForCforedListenPort"].as(); + else + g_config.ListenConf.CraneCtldForCforedListenPort = + kCtldForCforedDefaultPort; + + if (config["CraneCtldForCforedListenPort"]) + g_config.ListenConf.CraneCtldForCforedListenPort = + config["CraneCtldForCforedListenPort"].as(); + else + g_config.ListenConf.CraneCtldForCforedListenPort = + kCtldForCforedDefaultPort; + if (config["CompressedRpc"]) g_config.CompressedRpc = config["CompressedRpc"].as(); @@ -912,6 +928,9 @@ void InitializeCtldGlobalVariables() { g_ctld_for_craned_server = std::make_unique(g_config.ListenConf); + + g_ctld_for_cfored_server = + std::make_unique(g_config.ListenConf); } void CreateFolders() { @@ -945,6 +964,8 @@ int StartServer() { g_ctld_for_craned_server->Wait(); + g_ctld_for_cfored_server->Wait(); + DestroyCtldGlobalVariables(); return 0; diff --git a/src/CraneCtld/CranedKeeper.cpp b/src/CraneCtld/CranedKeeper.cpp index 9160ba741..39c433e39 100644 --- a/src/CraneCtld/CranedKeeper.cpp +++ b/src/CraneCtld/CranedKeeper.cpp @@ -729,14 +729,13 @@ void CranedKeeper::ConnectCranedNode_(CranedId const &craned_id) { ip_addr, g_config.CranedListenConf.CranedListenPort, m_channel_count_.fetch_add(1) + 1); - if (g_config.ListenConf.UseTls) { - SetTlsHostnameOverride(&channel_args, craned_id, - g_config.ListenConf.TlsCerts.DomainSuffix); - craned->m_channel_ = CreateTcpTlsCustomChannelByIp( - ip_addr, g_config.CranedListenConf.CranedListenPort, + if (g_config.ListenConf.UseTls) + craned->m_channel_ = CreateTcpTlsCustomChannelByHostname( + craned_id, g_config.CranedListenConf.CranedListenPort, g_config.ListenConf.TlsCerts.InternalCerts, - g_config.ListenConf.TlsCerts.CranedClientCerts, channel_args); - } else + g_config.ListenConf.TlsCerts.CranedClientCerts, + g_config.ListenConf.TlsCerts.DomainSuffix, channel_args); + else craned->m_channel_ = CreateTcpInsecureCustomChannel( ip_addr, g_config.CranedListenConf.CranedListenPort, channel_args); diff --git a/src/CraneCtld/CtldForCforedServer.cpp b/src/CraneCtld/CtldForCforedServer.cpp new file mode 100644 index 000000000..4415e39b7 --- /dev/null +++ b/src/CraneCtld/CtldForCforedServer.cpp @@ -0,0 +1,324 @@ +/** + * Copyright (c) 2023 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * CraneSched is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of + * the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, + * WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#include "CtldForCforedServer.h" + +#include +#include +#include + +#include "AccountManager.h" +#include "CranedKeeper.h" +#include "CranedMetaContainer.h" +#include "CtldGrpcServer.h" +#include "EmbeddedDbClient.h" +#include "TaskScheduler.h" +#include "crane/String.h" + +namespace Ctld { + +grpc::Status CtldForCforedServiceImpl::CforedStream( + grpc::ServerContext *context, + grpc::ServerReaderWriter *stream) { + using crane::grpc::InteractiveTaskType; + using crane::grpc::StreamCforedRequest; + using crane::grpc::StreamCtldReply; + using grpc::Status; + + enum class StreamState { + kWaitRegReq = 0, + kWaitMsg, + kCleanData, + }; + + bool ok; + + StreamCforedRequest cfored_request; + + auto stream_writer = std::make_shared(stream); + std::weak_ptr writer_weak_ptr(stream_writer); + std::string cfored_name; + + CRANE_TRACE("CforedStream from {} created.", context->peer()); + + StreamState state = StreamState::kWaitRegReq; + while (true) { + switch (state) { + case StreamState::kWaitRegReq: + ok = stream->Read(&cfored_request); + if (ok) { + if (cfored_request.type() != StreamCforedRequest::CFORED_REGISTRATION) { + CRANE_ERROR("Expect type CFORED_REGISTRATION from peer {}.", + context->peer()); + return Status::CANCELLED; + } else { + cfored_name = cfored_request.payload_cfored_reg().cfored_name(); + CRANE_INFO("Cfored {} registered.", cfored_name); + + ok = stream_writer->WriteCforedRegistrationAck({}); + if (ok) { + state = StreamState::kWaitMsg; + } else { + CRANE_ERROR( + "Failed to send msg to cfored {}. Connection is broken. " + "Exiting...", + cfored_name); + state = StreamState::kCleanData; + } + } + } else { + state = StreamState::kCleanData; + } + + break; + + case StreamState::kWaitMsg: { + ok = stream->Read(&cfored_request); + if (ok) { + switch (cfored_request.type()) { + case StreamCforedRequest::TASK_REQUEST: { + auto const &payload = cfored_request.payload_task_req(); + auto task = std::make_unique(); + task->SetFieldsByTaskToCtld(payload.task()); + + auto &meta = std::get(task->meta); + auto i_type = meta.interactive_type; + + meta.cb_task_res_allocated = + [writer_weak_ptr](task_id_t task_id, + std::string const &allocated_craned_regex, + std::list const &craned_ids) { + if (auto writer = writer_weak_ptr.lock(); writer) + writer->WriteTaskResAllocReply( + task_id, + {std::make_pair(allocated_craned_regex, craned_ids)}); + }; + + meta.cb_task_cancel = [writer_weak_ptr](task_id_t task_id) { + if (auto writer = writer_weak_ptr.lock(); writer) + writer->WriteTaskCancelRequest(task_id); + }; + + meta.cb_task_completed = [this, i_type, cfored_name, + writer_weak_ptr](task_id_t task_id) { + CRANE_TRACE("Sending TaskCompletionAckReply in task_completed", + task_id); + if (auto writer = writer_weak_ptr.lock(); writer) + writer->WriteTaskCompletionAckReply(task_id); + m_ctld_server_->m_mtx_.Lock(); + + // If cfored disconnected, the cfored_name should have be + // removed from the map and the task completion callback is + // generated from cleaning the remaining tasks by calling + // g_task_scheduler->TerminateTask(), we should ignore this + // callback since the task id has already been cleaned. + auto iter = + m_ctld_server_->m_cfored_running_tasks_.find(cfored_name); + if (iter != m_ctld_server_->m_cfored_running_tasks_.end()) + iter->second.erase(task_id); + m_ctld_server_->m_mtx_.Unlock(); + }; + + auto submit_result = + m_ctld_server_->SubmitTaskToScheduler(std::move(task)); + result::result result; + if (submit_result.has_value()) { + result = result::result{ + submit_result.value().get()}; + } else { + result = result::fail(submit_result.error()); + } + ok = stream_writer->WriteTaskIdReply(payload.pid(), result); + + if (!ok) { + CRANE_ERROR( + "Failed to send msg to cfored {}. Connection is broken. " + "Exiting...", + cfored_name); + state = StreamState::kCleanData; + } else { + if (result.has_value()) { + m_ctld_server_->m_mtx_.Lock(); + m_ctld_server_->m_cfored_running_tasks_[cfored_name].emplace( + result.value()); + m_ctld_server_->m_mtx_.Unlock(); + } + } + } break; + + case StreamCforedRequest::TASK_COMPLETION_REQUEST: { + auto const &payload = cfored_request.payload_task_complete_req(); + CRANE_TRACE("Recv TaskCompletionReq of Task #{}", payload.task_id()); + + if (g_task_scheduler->TerminatePendingOrRunningTask( + payload.task_id()) != CraneErr::kOk) + stream_writer->WriteTaskCompletionAckReply(payload.task_id()); + } break; + + case StreamCforedRequest::CFORED_GRACEFUL_EXIT: { + stream_writer->WriteCforedGracefulExitAck(); + stream_writer->Invalidate(); + state = StreamState::kCleanData; + } break; + + default: + CRANE_ERROR("Not expected cfored request type: {}", + StreamCforedRequest_CforedRequestType_Name( + cfored_request.type())); + return Status::CANCELLED; + } + } else { + state = StreamState::kCleanData; + } + } break; + + case StreamState::kCleanData: { + CRANE_INFO("Cfored {} disconnected. Cleaning its data...", cfored_name); + stream_writer->Invalidate(); + m_ctld_server_->m_mtx_.Lock(); + + auto const &running_task_set = + m_ctld_server_->m_cfored_running_tasks_[cfored_name]; + std::vector running_tasks(running_task_set.begin(), + running_task_set.end()); + m_ctld_server_->m_cfored_running_tasks_.erase(cfored_name); + m_ctld_server_->m_mtx_.Unlock(); + + for (task_id_t task_id : running_tasks) { + g_task_scheduler->TerminateRunningTask(task_id); + } + + return Status::OK; + } + } + } +} + +CtldForCforedServer::CtldForCforedServer(const Config::CraneCtldListenConf &listen_conf) { + m_service_impl_ = std::make_unique(this); + + grpc::ServerBuilder builder; + + if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); + + if (listen_conf.UseTls) + ServerBuilderAddmTcpTlsListeningPort( + &builder, listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForCforedListenPort, listen_conf.TlsCerts.InternalCerts, + listen_conf.TlsCerts.InternalCaContent); + else + ServerBuilderAddTcpInsecureListeningPort(&builder, + listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForCforedListenPort); + + builder.RegisterService(m_service_impl_.get()); + m_server_ = builder.BuildAndStart(); + if (!m_server_) { + CRANE_ERROR("Cannot start gRPC server!"); + std::exit(1); + } + CRANE_INFO("CraneCtld For Cfored Server is listening on {}:{} and Tls is {}", + listen_conf.CraneCtldListenAddr, listen_conf.CraneCtldForCforedListenPort, + listen_conf.UseTls); +} + +result::result, std::string> +CtldForCforedServer::SubmitTaskToScheduler(std::unique_ptr task) { + CraneErr err; + + if (!task->password_entry->Valid()) { + return result::fail( + fmt::format("Uid {} not found on the controller node", task->uid)); + } + task->SetUsername(task->password_entry->Username()); + + { // Limit the lifecycle of user_scoped_ptr + auto user_scoped_ptr = + g_account_manager->GetExistedUserInfo(task->Username()); + if (!user_scoped_ptr) { + return result::fail(fmt::format( + "User '{}' not found in the account database", task->Username())); + } + + if (task->account.empty()) { + task->account = user_scoped_ptr->default_account; + task->MutableTaskToCtld()->set_account(user_scoped_ptr->default_account); + } else { + if (!user_scoped_ptr->account_to_attrs_map.contains(task->account)) { + return result::fail(fmt::format( + "Account '{}' is not in your account list", task->account)); + } + } + } + + if (!g_account_manager->CheckUserPermissionToPartition( + task->Username(), task->account, task->partition_id)) { + return result::fail( + fmt::format("User '{}' doesn't have permission to use partition '{}' " + "when using account '{}'", + task->Username(), task->partition_id, task->account)); + } + + auto enable_res = + g_account_manager->CheckEnableState(task->account, task->Username()); + if (enable_res.has_error()) { + return result::fail(enable_res.error()); + } + + err = g_task_scheduler->AcquireTaskAttributes(task.get()); + + if (err == CraneErr::kOk) + err = g_task_scheduler->CheckTaskValidity(task.get()); + + if (err == CraneErr::kOk) { + task->SetSubmitTime(absl::Now()); + std::future future = + g_task_scheduler->SubmitTaskAsync(std::move(task)); + return {std::move(future)}; + } + + if (err == CraneErr::kNonExistent) { + CRANE_DEBUG("Task submission failed. Reason: Partition doesn't exist!"); + return result::fail("Partition doesn't exist!"); + } else if (err == CraneErr::kInvalidNodeNum) { + CRANE_DEBUG( + "Task submission failed. Reason: --node is either invalid or greater " + "than the number of nodes in its partition."); + return result::fail( + "--node is either invalid or greater than the number of nodes in its " + "partition."); + } else if (err == CraneErr::kNoResource) { + CRANE_DEBUG( + "Task submission failed. " + "Reason: The resources of the partition are insufficient."); + return result::fail("The resources of the partition are insufficient"); + } else if (err == CraneErr::kNoAvailNode) { + CRANE_DEBUG( + "Task submission failed. " + "Reason: Nodes satisfying the requirements of task are insufficient"); + return result::fail( + "Nodes satisfying the requirements of task are insufficient."); + } else if (err == CraneErr::kInvalidParam) { + CRANE_DEBUG( + "Task submission failed. " + "Reason: The param of task is invalid."); + return result::fail("The param of task is invalid."); + } + return result::fail(CraneErrStr(err)); +} + +} // namespace Ctld \ No newline at end of file diff --git a/src/CraneCtld/CtldForCforedServer.h b/src/CraneCtld/CtldForCforedServer.h new file mode 100644 index 000000000..c9389361e --- /dev/null +++ b/src/CraneCtld/CtldForCforedServer.h @@ -0,0 +1,215 @@ +/** + * Copyright (c) 2023 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * CraneSched is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of + * the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, + * WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#pragma once + +#include +#include "CtldForCranedServer.h" +#include "CtldPublicDefs.h" +// Precompiled header comes first! + +#include "crane/Lock.h" +#include "protos/Crane.grpc.pb.h" +#include "protos/Crane.pb.h" + +namespace Ctld { + +using crane::grpc::Craned; +using grpc::Channel; +using grpc::Server; + +class CforedStreamWriter { + private: + using Mutex = absl::Mutex; + using LockGuard = absl::MutexLock; + + using StreamCtldReply = crane::grpc::StreamCtldReply; + + public: + explicit CforedStreamWriter( + grpc::ServerReaderWriter *stream) + : m_stream_(stream), m_valid_(true) {} + + bool WriteTaskIdReply( + pid_t calloc_pid, + result::result res) { + LockGuard guard(&m_stream_mtx_); + if (!m_valid_) return false; + + StreamCtldReply reply; + reply.set_type(StreamCtldReply::TASK_ID_REPLY); + auto *task_id_reply = reply.mutable_payload_task_id_reply(); + if (res.has_value()) { + task_id_reply->set_ok(true); + task_id_reply->set_pid(calloc_pid); + task_id_reply->set_task_id(res.value()); + } else { + task_id_reply->set_ok(false); + task_id_reply->set_pid(calloc_pid); + task_id_reply->set_failure_reason(std::move(res.error())); + } + + return m_stream_->Write(reply); + } + + bool WriteTaskResAllocReply(task_id_t task_id, + result::result>, std::string> res) { + LockGuard guard(&m_stream_mtx_); + if (!m_valid_) return false; + + StreamCtldReply reply; + reply.set_type(StreamCtldReply::TASK_RES_ALLOC_REPLY); + auto *task_res_alloc_reply = reply.mutable_payload_task_res_alloc_reply(); + task_res_alloc_reply->set_task_id(task_id); + + if (res.has_value()) { + task_res_alloc_reply->set_ok(true); + task_res_alloc_reply->set_allocated_craned_regex(std::move(res.value().first)); + std::ranges::for_each(res.value().second,[&task_res_alloc_reply](const auto& craned_id){task_res_alloc_reply->add_craned_ids(craned_id);}); + } else { + task_res_alloc_reply->set_ok(false); + task_res_alloc_reply->set_failure_reason(std::move(res.error())); + } + + return m_stream_->Write(reply); + } + + bool WriteTaskCompletionAckReply(task_id_t task_id) { + LockGuard guard(&m_stream_mtx_); + if (!m_valid_) return false; + CRANE_TRACE("Sending TaskCompletionAckReply to cfored of task id {}",task_id); + StreamCtldReply reply; + reply.set_type(StreamCtldReply::TASK_COMPLETION_ACK_REPLY); + + auto *task_completion_ack = reply.mutable_payload_task_completion_ack(); + task_completion_ack->set_task_id(task_id); + + return m_stream_->Write(reply); + } + + bool WriteTaskCancelRequest(task_id_t task_id) { + LockGuard guard(&m_stream_mtx_); + if (!m_valid_) return false; + + StreamCtldReply reply; + reply.set_type(StreamCtldReply::TASK_CANCEL_REQUEST); + + auto *task_cancel_req = reply.mutable_payload_task_cancel_request(); + task_cancel_req->set_task_id(task_id); + + return m_stream_->Write(reply); + } + + bool WriteCforedRegistrationAck( + const result::result &res) { + LockGuard guard(&m_stream_mtx_); + if (!m_valid_) return false; + + StreamCtldReply reply; + reply.set_type(StreamCtldReply::CFORED_REGISTRATION_ACK); + + auto *cfored_reg_ack = reply.mutable_payload_cfored_reg_ack(); + if (res.has_value()) { + cfored_reg_ack->set_ok(true); + } else { + cfored_reg_ack->set_ok(false); + cfored_reg_ack->set_failure_reason(std::move(res.error())); + } + + return m_stream_->Write(reply); + } + + bool WriteCforedGracefulExitAck() { + LockGuard guard(&m_stream_mtx_); + if (!m_valid_) return false; + + StreamCtldReply reply; + reply.set_type(StreamCtldReply::CFORED_GRACEFUL_EXIT_ACK); + + auto *cfored_graceful_exit_ack = reply.mutable_payload_graceful_exit_ack(); + cfored_graceful_exit_ack->set_ok(true); + + return m_stream_->Write(reply); + } + + void Invalidate() { + LockGuard guard(&m_stream_mtx_); + m_valid_ = false; + } + + private: + Mutex m_stream_mtx_; + + bool m_valid_; + + grpc::ServerReaderWriter *m_stream_ + ABSL_GUARDED_BY(m_stream_mtx_); +}; + +class CtldForCforedServer; + +class CtldForCforedServiceImpl final : public crane::grpc::CraneCtldForCfored::Service { + public: + explicit CtldForCforedServiceImpl(CtldForCforedServer *server) : m_ctld_server_(server) {} + + grpc::Status CforedStream( + grpc::ServerContext *context, + grpc::ServerReaderWriter *stream) + override; + + private: + CtldForCforedServer *m_ctld_server_; +}; + +/*** + * Note: There should be only ONE instance of CtldServer!!!! + */ +class CtldForCforedServer { + public: + explicit CtldForCforedServer(const Config::CraneCtldListenConf &listen_conf); + + inline void Wait() { m_server_->Wait(); } + + result::result, std::string> SubmitTaskToScheduler( + std::unique_ptr task); + private: + template > + using HashMap = absl::flat_hash_map; + + template > + using HashSet = absl::flat_hash_set; + + using Mutex = util::mutex; + + std::unique_ptr m_service_impl_; + std::unique_ptr m_server_; + + Mutex m_mtx_; + HashMap> + m_cfored_running_tasks_ ABSL_GUARDED_BY(m_mtx_); + + friend class CtldForCforedServiceImpl; +}; + + +} // namespace Ctld + +inline std::unique_ptr g_ctld_for_cfored_server; \ No newline at end of file diff --git a/src/CraneCtld/CtldForCranedServer.cpp b/src/CraneCtld/CtldForCranedServer.cpp index be25e7f52..7ef450e4a 100644 --- a/src/CraneCtld/CtldForCranedServer.cpp +++ b/src/CraneCtld/CtldForCranedServer.cpp @@ -21,7 +21,6 @@ #include "CranedKeeper.h" #include "CranedMetaContainer.h" #include "TaskScheduler.h" -#include "crane/String.h" namespace Ctld { @@ -66,15 +65,15 @@ CtldForCranedServer::CtldForCranedServer(const Config::CraneCtldListenConf &list if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); - if (listen_conf.UseTls) { + if (listen_conf.UseTls) ServerBuilderAddmTcpTlsListeningPort( &builder, listen_conf.CraneCtldListenAddr, - listen_conf.CraneCtldForCranedListenPort, listen_conf.TlsCerts.InternalCerts, listen_conf.TlsCerts.InternalCaContent); - } else { - ServerBuilderAddTcpInsecureListeningPort(&builder, + listen_conf.CraneCtldForCranedListenPort, listen_conf.TlsCerts.InternalCerts, + listen_conf.TlsCerts.InternalCaContent); + else + ServerBuilderAddTcpInsecureListeningPort(&builder, listen_conf.CraneCtldListenAddr, listen_conf.CraneCtldForCranedListenPort); - } builder.RegisterService(m_service_impl_.get()); m_server_ = builder.BuildAndStart(); diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index 8904d10a8..de06d9a27 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -675,185 +675,6 @@ grpc::Status CraneCtldServiceImpl::QueryClusterInfo( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::CforedStream( - grpc::ServerContext *context, - grpc::ServerReaderWriter *stream) { - using crane::grpc::InteractiveTaskType; - using crane::grpc::StreamCforedRequest; - using crane::grpc::StreamCtldReply; - using grpc::Status; - - enum class StreamState { - kWaitRegReq = 0, - kWaitMsg, - kCleanData, - }; - - bool ok; - - StreamCforedRequest cfored_request; - - auto stream_writer = std::make_shared(stream); - std::weak_ptr writer_weak_ptr(stream_writer); - std::string cfored_name; - - CRANE_TRACE("CforedStream from {} created.", context->peer()); - - StreamState state = StreamState::kWaitRegReq; - while (true) { - switch (state) { - case StreamState::kWaitRegReq: - ok = stream->Read(&cfored_request); - if (ok) { - if (cfored_request.type() != StreamCforedRequest::CFORED_REGISTRATION) { - CRANE_ERROR("Expect type CFORED_REGISTRATION from peer {}.", - context->peer()); - return Status::CANCELLED; - } - - cfored_name = cfored_request.payload_cfored_reg().cfored_name(); - CRANE_INFO("Cfored {} registered.", cfored_name); - - ok = stream_writer->WriteCforedRegistrationAck({}); - if (ok) { - state = StreamState::kWaitMsg; - } else { - CRANE_ERROR( - "Failed to send msg to cfored {}. Connection is broken. " - "Exiting...", - cfored_name); - state = StreamState::kCleanData; - } - - } else { - state = StreamState::kCleanData; - } - - break; - - case StreamState::kWaitMsg: { - ok = stream->Read(&cfored_request); - if (ok) { - switch (cfored_request.type()) { - case StreamCforedRequest::TASK_REQUEST: { - auto const &payload = cfored_request.payload_task_req(); - auto task = std::make_unique(); - task->SetFieldsByTaskToCtld(payload.task()); - - auto &meta = std::get(task->meta); - auto i_type = meta.interactive_type; - - meta.cb_task_res_allocated = - [writer_weak_ptr](task_id_t task_id, - std::string const &allocated_craned_regex, - std::list const &craned_ids) { - if (auto writer = writer_weak_ptr.lock(); writer) - writer->WriteTaskResAllocReply( - task_id, - {std::make_pair(allocated_craned_regex, craned_ids)}); - }; - - meta.cb_task_cancel = [writer_weak_ptr](task_id_t task_id) { - CRANE_TRACE("Sending TaskCancelRequest in task_cancel", task_id); - if (auto writer = writer_weak_ptr.lock(); writer) - writer->WriteTaskCancelRequest(task_id); - }; - - meta.cb_task_completed = [this, i_type, cfored_name, writer_weak_ptr]( - task_id_t task_id, - bool send_completion_ack) { - if (auto writer = writer_weak_ptr.lock(); - writer && send_completion_ack) - writer->WriteTaskCompletionAckReply(task_id); - m_ctld_server_->m_mtx_.Lock(); - - // If cfored disconnected, the cfored_name should have be - // removed from the map and the task completion callback is - // generated from cleaning the remaining tasks by calling - // g_task_scheduler->TerminateTask(), we should ignore this - // callback since the task id has already been cleaned. - auto iter = - m_ctld_server_->m_cfored_running_tasks_.find(cfored_name); - if (iter != m_ctld_server_->m_cfored_running_tasks_.end()) - iter->second.erase(task_id); - m_ctld_server_->m_mtx_.Unlock(); - }; - - auto submit_result = - m_ctld_server_->SubmitTaskToScheduler(std::move(task)); - std::expected result; - if (submit_result.has_value()) { - result = std::expected{ - submit_result.value().get()}; - } else { - result = std::unexpected(submit_result.error()); - } - ok = stream_writer->WriteTaskIdReply(payload.pid(), result); - - if (!ok) { - CRANE_ERROR( - "Failed to send msg to cfored {}. Connection is broken. " - "Exiting...", - cfored_name); - state = StreamState::kCleanData; - } else { - if (result.has_value()) { - m_ctld_server_->m_mtx_.Lock(); - m_ctld_server_->m_cfored_running_tasks_[cfored_name].emplace( - result.value()); - m_ctld_server_->m_mtx_.Unlock(); - } - } - } break; - - case StreamCforedRequest::TASK_COMPLETION_REQUEST: { - auto const &payload = cfored_request.payload_task_complete_req(); - CRANE_TRACE("Recv TaskCompletionReq of Task #{}", payload.task_id()); - if (g_task_scheduler->TerminatePendingOrRunningIaTask( - payload.task_id()) != CraneErr::kOk) - stream_writer->WriteTaskCompletionAckReply(payload.task_id()); - } break; - - case StreamCforedRequest::CFORED_GRACEFUL_EXIT: { - stream_writer->WriteCforedGracefulExitAck(); - stream_writer->Invalidate(); - state = StreamState::kCleanData; - } break; - - default: - CRANE_ERROR("Not expected cfored request type: {}", - StreamCforedRequest_CforedRequestType_Name( - cfored_request.type())); - return Status::CANCELLED; - } - } else { - state = StreamState::kCleanData; - } - } break; - - case StreamState::kCleanData: { - CRANE_INFO("Cfored {} disconnected. Cleaning its data...", cfored_name); - stream_writer->Invalidate(); - m_ctld_server_->m_mtx_.Lock(); - - auto const &running_task_set = - m_ctld_server_->m_cfored_running_tasks_[cfored_name]; - std::vector running_tasks(running_task_set.begin(), - running_task_set.end()); - m_ctld_server_->m_cfored_running_tasks_.erase(cfored_name); - m_ctld_server_->m_mtx_.Unlock(); - - for (task_id_t task_id : running_tasks) { - g_task_scheduler->TerminateRunningTask(task_id); - } - - return Status::OK; - } - } - } -} - CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { m_service_impl_ = std::make_unique(this); diff --git a/src/CraneCtld/CtldGrpcServer.h b/src/CraneCtld/CtldGrpcServer.h index 59255b53a..18e877a8f 100644 --- a/src/CraneCtld/CtldGrpcServer.h +++ b/src/CraneCtld/CtldGrpcServer.h @@ -31,153 +31,12 @@ using crane::grpc::Craned; using grpc::Channel; using grpc::Server; -class CforedStreamWriter { - private: - using Mutex = absl::Mutex; - using LockGuard = absl::MutexLock; - - using StreamCtldReply = crane::grpc::StreamCtldReply; - - public: - explicit CforedStreamWriter( - grpc::ServerReaderWriter *stream) - : m_stream_(stream), m_valid_(true) {} - - bool WriteTaskIdReply(pid_t calloc_pid, - std::expected res) { - LockGuard guard(&m_stream_mtx_); - if (!m_valid_) return false; - - StreamCtldReply reply; - reply.set_type(StreamCtldReply::TASK_ID_REPLY); - auto *task_id_reply = reply.mutable_payload_task_id_reply(); - if (res.has_value()) { - task_id_reply->set_ok(true); - task_id_reply->set_pid(calloc_pid); - task_id_reply->set_task_id(res.value()); - } else { - task_id_reply->set_ok(false); - task_id_reply->set_pid(calloc_pid); - task_id_reply->set_failure_reason(std::move(res.error())); - } - - return m_stream_->Write(reply); - } - - bool WriteTaskResAllocReply( - task_id_t task_id, - std::expected>, std::string> - res) { - LockGuard guard(&m_stream_mtx_); - if (!m_valid_) return false; - - StreamCtldReply reply; - reply.set_type(StreamCtldReply::TASK_RES_ALLOC_REPLY); - auto *task_res_alloc_reply = reply.mutable_payload_task_res_alloc_reply(); - task_res_alloc_reply->set_task_id(task_id); - - if (res.has_value()) { - task_res_alloc_reply->set_ok(true); - task_res_alloc_reply->set_allocated_craned_regex( - std::move(res.value().first)); - std::ranges::for_each(res.value().second, - [&task_res_alloc_reply](const auto &craned_id) { - task_res_alloc_reply->add_craned_ids(craned_id); - }); - } else { - task_res_alloc_reply->set_ok(false); - task_res_alloc_reply->set_failure_reason(std::move(res.error())); - } - - return m_stream_->Write(reply); - } - - bool WriteTaskCompletionAckReply(task_id_t task_id) { - LockGuard guard(&m_stream_mtx_); - if (!m_valid_) return false; - CRANE_TRACE("Sending TaskCompletionAckReply to cfored of task id {}", - task_id); - StreamCtldReply reply; - reply.set_type(StreamCtldReply::TASK_COMPLETION_ACK_REPLY); - - auto *task_completion_ack = reply.mutable_payload_task_completion_ack(); - task_completion_ack->set_task_id(task_id); - - return m_stream_->Write(reply); - } - - bool WriteTaskCancelRequest(task_id_t task_id) { - LockGuard guard(&m_stream_mtx_); - if (!m_valid_) return false; - - StreamCtldReply reply; - reply.set_type(StreamCtldReply::TASK_CANCEL_REQUEST); - - auto *task_cancel_req = reply.mutable_payload_task_cancel_request(); - task_cancel_req->set_task_id(task_id); - - return m_stream_->Write(reply); - } - - bool WriteCforedRegistrationAck(const std::expected &res) { - LockGuard guard(&m_stream_mtx_); - if (!m_valid_) return false; - - StreamCtldReply reply; - reply.set_type(StreamCtldReply::CFORED_REGISTRATION_ACK); - - auto *cfored_reg_ack = reply.mutable_payload_cfored_reg_ack(); - if (res.has_value()) { - cfored_reg_ack->set_ok(true); - } else { - cfored_reg_ack->set_ok(false); - cfored_reg_ack->set_failure_reason(std::move(res.error())); - } - - return m_stream_->Write(reply); - } - - bool WriteCforedGracefulExitAck() { - LockGuard guard(&m_stream_mtx_); - if (!m_valid_) return false; - - StreamCtldReply reply; - reply.set_type(StreamCtldReply::CFORED_GRACEFUL_EXIT_ACK); - - auto *cfored_graceful_exit_ack = reply.mutable_payload_graceful_exit_ack(); - cfored_graceful_exit_ack->set_ok(true); - - return m_stream_->Write(reply); - } - - void Invalidate() { - LockGuard guard(&m_stream_mtx_); - m_valid_ = false; - } - - private: - Mutex m_stream_mtx_; - - bool m_valid_; - - grpc::ServerReaderWriter *m_stream_ - ABSL_GUARDED_BY(m_stream_mtx_); -}; - class CtldServer; class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { public: explicit CraneCtldServiceImpl(CtldServer *server) : m_ctld_server_(server) {} - grpc::Status CforedStream( - grpc::ServerContext *context, - grpc::ServerReaderWriter *stream) - override; - grpc::Status SubmitBatchTask( grpc::ServerContext *context, const crane::grpc::SubmitBatchTaskRequest *request, @@ -300,23 +159,9 @@ class CtldServer { std::unique_ptr task); private: - template > - using HashMap = absl::flat_hash_map; - - template > - using HashSet = absl::flat_hash_set; - - using Mutex = util::mutex; - std::unique_ptr m_service_impl_; std::unique_ptr m_server_; - Mutex m_mtx_; - HashMap> - m_cfored_running_tasks_ ABSL_GUARDED_BY(m_mtx_); - inline static std::mutex s_sigint_mtx; inline static std::condition_variable s_sigint_cv; static void signal_handler_func(int) { s_sigint_cv.notify_one(); }; diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 3bfbc053e..06aa0fc7d 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -88,6 +88,7 @@ struct Config { std::string CraneCtldListenAddr; std::string CraneCtldListenPort; std::string CraneCtldForCranedListenPort; + std::string CraneCtldForCforedListenPort; bool UseTls{false}; struct TlsCertsConfig { diff --git a/src/Craned/CforedClient.cpp b/src/Craned/CforedClient.cpp index 8a3649a4b..bc946b71f 100644 --- a/src/Craned/CforedClient.cpp +++ b/src/Craned/CforedClient.cpp @@ -20,6 +20,7 @@ #include +#include "CranedPublicDefs.h" #include "crane/String.h" namespace Craned { @@ -43,14 +44,15 @@ void CforedClient::InitChannelAndStub(const std::string& cfored_name) { channel_args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP); // Todo: Use cfored listen config - if (g_config.ListenConf.UseTls) { - // m_cfored_channel_ = CreateTcpTlsChannelByHostname( - // cfored_name, kCforedDefaultPort, g_config.ListenConf.TlsCerts); - CreateTcpInsecureChannel(cfored_name, kCforedDefaultPort); - } else { + if (g_config.ListenConf.UseTls) + m_cfored_channel_ = CreateTcpTlsChannelByHostname( + cfored_name, kCforedDefaultPort, + g_config.ListenConf.TlsCerts.CranedTlsCerts, + g_config.ListenConf.TlsCerts.CforedClientTlsCerts, + g_config.ListenConf.TlsCerts.DomainSuffix); + else m_cfored_channel_ = CreateTcpInsecureChannel(cfored_name, kCforedDefaultPort); - } // std::unique_ptr will automatically release the dangling stub. m_stub_ = crane::grpc::CraneForeD::NewStub(m_cfored_channel_); diff --git a/src/Craned/Craned.cpp b/src/Craned/Craned.cpp index dadf8cfe8..e7175c3d2 100644 --- a/src/Craned/Craned.cpp +++ b/src/Craned/Craned.cpp @@ -178,6 +178,8 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.TlsCerts.CranedTlsCerts; ClientTlsCertificates& internal_client_certs = g_config.ListenConf.TlsCerts.InternalClientTlsCerts; + ClientTlsCertificates& cfoed_client_certs = + g_config.ListenConf.TlsCerts.CforedClientTlsCerts; if (config["DomainSuffix"]) g_config.ListenConf.TlsCerts.DomainSuffix = @@ -268,6 +270,29 @@ void ParseConfig(int argc, char** argv) { "UseTls is true, but CranectldInternalCertFilePath is empty"); std::exit(1); } + + if (config["CforedCertFilePath"]) { + cfoed_client_certs.ClientCertFilePath = + config["CforedCertFilePath"].as(); + + try { + cfoed_client_certs.ClientCertContent = + util::ReadFileIntoString(cfoed_client_certs.ClientCertFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (cfoed_client_certs.ClientCertContent.empty()) { + CRANE_ERROR( + "UseTls is true, but the file specified by CforedCertFilePath " + "is empty"); + } + } else { + CRANE_ERROR( + "UseTls is true, but CranectldInternalCertFilePath is empty"); + std::exit(1); + } + } else { g_config.ListenConf.UseTls = false; } diff --git a/src/Craned/CranedPublicDefs.h b/src/Craned/CranedPublicDefs.h index 5bd3c1620..323fadaac 100644 --- a/src/Craned/CranedPublicDefs.h +++ b/src/Craned/CranedPublicDefs.h @@ -60,6 +60,7 @@ struct Config { std::string DomainSuffix; TlsCertificates CranedTlsCerts; ClientTlsCertificates InternalClientTlsCerts; + ClientTlsCertificates CforedClientTlsCerts; }; TlsCertsConfig TlsCerts; From 1bb675944dd9db6c7afa29872265d7a6b29317c9 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:19:17 +0800 Subject: [PATCH 04/62] feat: Move the SubmitTaskToScheduler function to TaskScheduler. --- src/CraneCtld/CtldForCforedServer.cpp | 88 +------------------------- src/CraneCtld/CtldForCforedServer.h | 3 - src/CraneCtld/CtldGrpcServer.cpp | 89 +-------------------------- src/CraneCtld/CtldGrpcServer.h | 3 - src/CraneCtld/TaskScheduler.cpp | 85 +++++++++++++++++++++++++ src/CraneCtld/TaskScheduler.h | 3 + 6 files changed, 92 insertions(+), 179 deletions(-) diff --git a/src/CraneCtld/CtldForCforedServer.cpp b/src/CraneCtld/CtldForCforedServer.cpp index 4415e39b7..450804063 100644 --- a/src/CraneCtld/CtldForCforedServer.cpp +++ b/src/CraneCtld/CtldForCforedServer.cpp @@ -24,6 +24,7 @@ #include "CranedKeeper.h" #include "CranedMetaContainer.h" #include "CtldGrpcServer.h" +#include "CtldPublicDefs.h" #include "EmbeddedDbClient.h" #include "TaskScheduler.h" #include "crane/String.h" @@ -134,7 +135,7 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( }; auto submit_result = - m_ctld_server_->SubmitTaskToScheduler(std::move(task)); + g_task_scheduler->SubmitTaskToScheduler(std::move(task)); result::result result; if (submit_result.has_value()) { result = result::result{ @@ -236,89 +237,4 @@ CtldForCforedServer::CtldForCforedServer(const Config::CraneCtldListenConf &list listen_conf.UseTls); } -result::result, std::string> -CtldForCforedServer::SubmitTaskToScheduler(std::unique_ptr task) { - CraneErr err; - - if (!task->password_entry->Valid()) { - return result::fail( - fmt::format("Uid {} not found on the controller node", task->uid)); - } - task->SetUsername(task->password_entry->Username()); - - { // Limit the lifecycle of user_scoped_ptr - auto user_scoped_ptr = - g_account_manager->GetExistedUserInfo(task->Username()); - if (!user_scoped_ptr) { - return result::fail(fmt::format( - "User '{}' not found in the account database", task->Username())); - } - - if (task->account.empty()) { - task->account = user_scoped_ptr->default_account; - task->MutableTaskToCtld()->set_account(user_scoped_ptr->default_account); - } else { - if (!user_scoped_ptr->account_to_attrs_map.contains(task->account)) { - return result::fail(fmt::format( - "Account '{}' is not in your account list", task->account)); - } - } - } - - if (!g_account_manager->CheckUserPermissionToPartition( - task->Username(), task->account, task->partition_id)) { - return result::fail( - fmt::format("User '{}' doesn't have permission to use partition '{}' " - "when using account '{}'", - task->Username(), task->partition_id, task->account)); - } - - auto enable_res = - g_account_manager->CheckEnableState(task->account, task->Username()); - if (enable_res.has_error()) { - return result::fail(enable_res.error()); - } - - err = g_task_scheduler->AcquireTaskAttributes(task.get()); - - if (err == CraneErr::kOk) - err = g_task_scheduler->CheckTaskValidity(task.get()); - - if (err == CraneErr::kOk) { - task->SetSubmitTime(absl::Now()); - std::future future = - g_task_scheduler->SubmitTaskAsync(std::move(task)); - return {std::move(future)}; - } - - if (err == CraneErr::kNonExistent) { - CRANE_DEBUG("Task submission failed. Reason: Partition doesn't exist!"); - return result::fail("Partition doesn't exist!"); - } else if (err == CraneErr::kInvalidNodeNum) { - CRANE_DEBUG( - "Task submission failed. Reason: --node is either invalid or greater " - "than the number of nodes in its partition."); - return result::fail( - "--node is either invalid or greater than the number of nodes in its " - "partition."); - } else if (err == CraneErr::kNoResource) { - CRANE_DEBUG( - "Task submission failed. " - "Reason: The resources of the partition are insufficient."); - return result::fail("The resources of the partition are insufficient"); - } else if (err == CraneErr::kNoAvailNode) { - CRANE_DEBUG( - "Task submission failed. " - "Reason: Nodes satisfying the requirements of task are insufficient"); - return result::fail( - "Nodes satisfying the requirements of task are insufficient."); - } else if (err == CraneErr::kInvalidParam) { - CRANE_DEBUG( - "Task submission failed. " - "Reason: The param of task is invalid."); - return result::fail("The param of task is invalid."); - } - return result::fail(CraneErrStr(err)); -} - } // namespace Ctld \ No newline at end of file diff --git a/src/CraneCtld/CtldForCforedServer.h b/src/CraneCtld/CtldForCforedServer.h index c9389361e..ac3c09b77 100644 --- a/src/CraneCtld/CtldForCforedServer.h +++ b/src/CraneCtld/CtldForCforedServer.h @@ -185,9 +185,6 @@ class CtldForCforedServer { explicit CtldForCforedServer(const Config::CraneCtldListenConf &listen_conf); inline void Wait() { m_server_->Wait(); } - - result::result, std::string> SubmitTaskToScheduler( - std::unique_ptr task); private: template > diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index de06d9a27..794380a8a 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -33,7 +33,7 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTask( auto task = std::make_unique(); task->SetFieldsByTaskToCtld(request->task()); - auto result = m_ctld_server_->SubmitTaskToScheduler(std::move(task)); + auto result = g_task_scheduler->SubmitTaskToScheduler(std::move(task)); if (result.has_value()) { task_id_t id = result.value().get(); if (id != 0) { @@ -67,7 +67,7 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTasks( auto task = std::make_unique(); task->SetFieldsByTaskToCtld(task_to_ctld); - auto result = m_ctld_server_->SubmitTaskToScheduler(std::move(task)); + auto result = g_task_scheduler->SubmitTaskToScheduler(std::move(task)); results.emplace_back(std::move(result)); } @@ -728,89 +728,4 @@ CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { signal(SIGINT, &CtldServer::signal_handler_func); } -std::expected, std::string> -CtldServer::SubmitTaskToScheduler(std::unique_ptr task) { - CraneErr err; - - if (!task->password_entry->Valid()) { - return std::unexpected( - fmt::format("Uid {} not found on the controller node", task->uid)); - } - task->SetUsername(task->password_entry->Username()); - - { // Limit the lifecycle of user_scoped_ptr - auto user_scoped_ptr = - g_account_manager->GetExistedUserInfo(task->Username()); - if (!user_scoped_ptr) { - return std::unexpected(fmt::format( - "User '{}' not found in the account database", task->Username())); - } - - if (task->account.empty()) { - task->account = user_scoped_ptr->default_account; - task->MutableTaskToCtld()->set_account(user_scoped_ptr->default_account); - } else { - if (!user_scoped_ptr->account_to_attrs_map.contains(task->account)) { - return std::unexpected(fmt::format( - "Account '{}' is not in your account list", task->account)); - } - } - } - - if (!g_account_manager->CheckUserPermissionToPartition( - task->Username(), task->account, task->partition_id)) { - return std::unexpected( - fmt::format("User '{}' doesn't have permission to use partition '{}' " - "when using account '{}'", - task->Username(), task->partition_id, task->account)); - } - - auto enable_res = g_account_manager->CheckIfUserOfAccountIsEnabled( - task->Username(), task->account); - if (!enable_res) { - return std::unexpected(enable_res.error()); - } - - err = g_task_scheduler->AcquireTaskAttributes(task.get()); - - if (err == CraneErr::kOk) - err = g_task_scheduler->CheckTaskValidity(task.get()); - - if (err == CraneErr::kOk) { - task->SetSubmitTime(absl::Now()); - std::future future = - g_task_scheduler->SubmitTaskAsync(std::move(task)); - return {std::move(future)}; - } - - if (err == CraneErr::kNonExistent) { - CRANE_DEBUG("Task submission failed. Reason: Partition doesn't exist!"); - return std::unexpected("Partition doesn't exist!"); - } else if (err == CraneErr::kInvalidNodeNum) { - CRANE_DEBUG( - "Task submission failed. Reason: --node is either invalid or greater " - "than the number of nodes in its partition."); - return std::unexpected( - "--node is either invalid or greater than the number of nodes in its " - "partition."); - } else if (err == CraneErr::kNoResource) { - CRANE_DEBUG( - "Task submission failed. " - "Reason: The resources of the partition are insufficient."); - return std::unexpected("The resources of the partition are insufficient"); - } else if (err == CraneErr::kNoAvailNode) { - CRANE_DEBUG( - "Task submission failed. " - "Reason: Nodes satisfying the requirements of task are insufficient"); - return std::unexpected( - "Nodes satisfying the requirements of task are insufficient."); - } else if (err == CraneErr::kInvalidParam) { - CRANE_DEBUG( - "Task submission failed. " - "Reason: The param of task is invalid."); - return std::unexpected("The param of task is invalid."); - } - return std::unexpected(CraneErrStr(err)); -} - } // namespace Ctld diff --git a/src/CraneCtld/CtldGrpcServer.h b/src/CraneCtld/CtldGrpcServer.h index 18e877a8f..631b356aa 100644 --- a/src/CraneCtld/CtldGrpcServer.h +++ b/src/CraneCtld/CtldGrpcServer.h @@ -155,9 +155,6 @@ class CtldServer { inline void Wait() { m_server_->Wait(); } - std::expected, std::string> SubmitTaskToScheduler( - std::unique_ptr task); - private: std::unique_ptr m_service_impl_; std::unique_ptr m_server_; diff --git a/src/CraneCtld/TaskScheduler.cpp b/src/CraneCtld/TaskScheduler.cpp index fbb2d713f..e7e0a614f 100644 --- a/src/CraneCtld/TaskScheduler.cpp +++ b/src/CraneCtld/TaskScheduler.cpp @@ -1041,6 +1041,91 @@ void TaskScheduler::SetNodeSelectionAlgo( m_node_selection_algo_ = std::move(algo); } +result::result, std::string> +TaskScheduler::SubmitTaskToScheduler(std::unique_ptr task) { + CraneErr err; + + if (!task->password_entry->Valid()) { + return result::fail( + fmt::format("Uid {} not found on the controller node", task->uid)); + } + task->SetUsername(task->password_entry->Username()); + + { // Limit the lifecycle of user_scoped_ptr + auto user_scoped_ptr = + g_account_manager->GetExistedUserInfo(task->Username()); + if (!user_scoped_ptr) { + return result::fail(fmt::format( + "User '{}' not found in the account database", task->Username())); + } + + if (task->account.empty()) { + task->account = user_scoped_ptr->default_account; + task->MutableTaskToCtld()->set_account(user_scoped_ptr->default_account); + } else { + if (!user_scoped_ptr->account_to_attrs_map.contains(task->account)) { + return result::fail(fmt::format( + "Account '{}' is not in your account list", task->account)); + } + } + } + + if (!g_account_manager->CheckUserPermissionToPartition( + task->Username(), task->account, task->partition_id)) { + return result::fail( + fmt::format("User '{}' doesn't have permission to use partition '{}' " + "when using account '{}'", + task->Username(), task->partition_id, task->account)); + } + + auto enable_res = + g_account_manager->CheckEnableState(task->account, task->Username()); + if (enable_res.has_error()) { + return result::fail(enable_res.error()); + } + + err = AcquireTaskAttributes(task.get()); + + if (err == CraneErr::kOk) + err = CheckTaskValidity(task.get()); + + if (err == CraneErr::kOk) { + task->SetSubmitTime(absl::Now()); + std::future future = + SubmitTaskAsync(std::move(task)); + return {std::move(future)}; + } + + if (err == CraneErr::kNonExistent) { + CRANE_DEBUG("Task submission failed. Reason: Partition doesn't exist!"); + return result::fail("Partition doesn't exist!"); + } else if (err == CraneErr::kInvalidNodeNum) { + CRANE_DEBUG( + "Task submission failed. Reason: --node is either invalid or greater " + "than the number of nodes in its partition."); + return result::fail( + "--node is either invalid or greater than the number of nodes in its " + "partition."); + } else if (err == CraneErr::kNoResource) { + CRANE_DEBUG( + "Task submission failed. " + "Reason: The resources of the partition are insufficient."); + return result::fail("The resources of the partition are insufficient"); + } else if (err == CraneErr::kNoAvailNode) { + CRANE_DEBUG( + "Task submission failed. " + "Reason: Nodes satisfying the requirements of task are insufficient"); + return result::fail( + "Nodes satisfying the requirements of task are insufficient."); + } else if (err == CraneErr::kInvalidParam) { + CRANE_DEBUG( + "Task submission failed. " + "Reason: The param of task is invalid."); + return result::fail("The param of task is invalid."); + } + return result::fail(CraneErrStr(err)); +} + std::future TaskScheduler::SubmitTaskAsync( std::unique_ptr task) { std::promise promise; diff --git a/src/CraneCtld/TaskScheduler.h b/src/CraneCtld/TaskScheduler.h index fb202982e..efe0567a9 100644 --- a/src/CraneCtld/TaskScheduler.h +++ b/src/CraneCtld/TaskScheduler.h @@ -445,6 +445,9 @@ class TaskScheduler { void SetNodeSelectionAlgo(std::unique_ptr algo); + result::result, std::string> SubmitTaskToScheduler( + std::unique_ptr task); + /// \return The future is set to 0 if task submission is failed. /// Otherwise, it is set to newly allocated task id. std::future SubmitTaskAsync(std::unique_ptr task); From 17562341c371fd1a214eabe93656fbcff28c8e0b Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 21 Oct 2024 09:29:32 +0800 Subject: [PATCH 05/62] feat: update config.yaml --- etc/config.yaml | 21 ++++++++++++++++----- src/CraneCtld/CtldForCranedServer.h | 15 --------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/etc/config.yaml b/etc/config.yaml index 4a1a8d6ec..c01fc5149 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -11,17 +11,28 @@ DbConfigPath: /etc/crane/database.yaml CraneBaseDir: /var/crane/ # Tls settings -UseTls: false -ServerCertFilePath: /etc/crane/server.crt -ServerKeyFilePath: /etc/crane/server.key -CaCertFilePath: /etc/crane/ca.crt -DomainSuffix: riley.local +UseTls: true +CranectldExternalCertFilePath: /etc/crane/cranectld_external.pem +CranectldExternalKeyFilePath: /etc/crane/cranectld_external.key +ExternalCaFilePath: /etc/crane/external_ca.pem + +CranectldInternalCertFilePath: /etc/crane/cranectld_internal.pem +CranectldInternalKeyFilePath: /etc/crane/cranectld_internal.key +CranedCertFilePath: /etc/crane/craned.pem +CranedKeyFilePath: /etc/crane/craned.key +CforedCertFilePath: /etc/crane/cfored.pem +CforedKeyFilePath: /etc/crane/cfored.key +InternalCaFilePath: /etc/crane/internal_ca.pem + +DomainSuffix: crane.com # Ctld settings # the listening address of control machine CraneCtldListenAddr: 0.0.0.0 # the port of control machine to listen CraneCtldListenPort: 10011 +CraneCtldForCranedListenPort: 10013 +CraneCtldForCforedListenPort: 10014 # debug level of cranectld CraneCtldDebugLevel: trace # file path of cranectld log file (relative to CraneBaseDir) diff --git a/src/CraneCtld/CtldForCranedServer.h b/src/CraneCtld/CtldForCranedServer.h index c29af8fbb..74f89f7e7 100644 --- a/src/CraneCtld/CtldForCranedServer.h +++ b/src/CraneCtld/CtldForCranedServer.h @@ -19,7 +19,6 @@ #include "CtldPublicDefs.h" // Precompiled header comes first! -#include "crane/Lock.h" #include "protos/Crane.grpc.pb.h" #include "protos/Crane.pb.h" @@ -58,23 +57,9 @@ class CtldForCranedServer { inline void Wait() { m_server_->Wait(); } private: - template > - using HashMap = absl::flat_hash_map; - - template > - using HashSet = absl::flat_hash_set; - - using Mutex = util::mutex; - std::unique_ptr m_service_impl_; std::unique_ptr m_server_; - inline static std::mutex s_sigint_mtx; - inline static std::condition_variable s_sigint_cv; - static void signal_handler_func(int) { s_sigint_cv.notify_one(); }; - friend class CtldForCranedServiceImpl; }; From acb1a60b44f785051645d153a447f11a14f0c4c9 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 31 Oct 2024 17:11:13 +0800 Subject: [PATCH 06/62] feat: jwt util --- dependencies/cmake/CMakeLists.txt | 3 +- dependencies/cmake/jwt-cpp/CMakeLists.txt | 13 ++++++ src/Utilities/PublicHeader/CMakeLists.txt | 4 +- .../PublicHeader/include/crane/GrpcHelper.h | 39 ++++++++++++++---- .../PublicHeader/include/crane/Jwt.h | 21 ++++++++++ src/Utilities/PublicHeader/jwt.cpp | 41 +++++++++++++++++++ 6 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 dependencies/cmake/jwt-cpp/CMakeLists.txt create mode 100644 src/Utilities/PublicHeader/include/crane/Jwt.h create mode 100644 src/Utilities/PublicHeader/jwt.cpp diff --git a/dependencies/cmake/CMakeLists.txt b/dependencies/cmake/CMakeLists.txt index 7a0cf013e..bc69f9a11 100644 --- a/dependencies/cmake/CMakeLists.txt +++ b/dependencies/cmake/CMakeLists.txt @@ -13,7 +13,8 @@ if(CRANE_USE_MIMALLOC) add_subdirectory(mimalloc) endif() add_subdirectory(BSThreadPool) -add_subdirectory(nlohmann_json) +# add_subdirectory(nlohmann_json) +add_subdirectory(jwt-cpp) add_subdirectory(yaml-cpp) add_subdirectory(fmt) add_subdirectory(googletest) diff --git a/dependencies/cmake/jwt-cpp/CMakeLists.txt b/dependencies/cmake/jwt-cpp/CMakeLists.txt new file mode 100644 index 000000000..cc2afc5ff --- /dev/null +++ b/dependencies/cmake/jwt-cpp/CMakeLists.txt @@ -0,0 +1,13 @@ +include(FetchContent) + + +set(JWT_CPP_SRC_URL "https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz") + +fetchcontent_declare(jwt-cpp + URL ${JWT_CPP_SRC_URL} + URL_HASH SHA256=b9eb270e3ba8221e4b2bc38723c9a1cb4fa6c241a42908b9a334daff31137406 + INACTIVITY_TIMEOUT 5 + ) +set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "disable building examples" FORCE) + +fetchcontent_makeavailable(jwt-cpp) \ No newline at end of file diff --git a/src/Utilities/PublicHeader/CMakeLists.txt b/src/Utilities/PublicHeader/CMakeLists.txt index decb8e5e3..e6e88b32b 100644 --- a/src/Utilities/PublicHeader/CMakeLists.txt +++ b/src/Utilities/PublicHeader/CMakeLists.txt @@ -1,5 +1,5 @@ add_library(Utility_PublicHeader - String.cpp Network.cpp OS.cpp PublicHeader.cpp Logger.cpp + String.cpp Network.cpp OS.cpp PublicHeader.cpp Logger.cpp jwt.cpp include/crane/String.h include/crane/Network.h include/crane/OS.h @@ -7,6 +7,7 @@ add_library(Utility_PublicHeader include/crane/Lock.h include/crane/Pointer.h include/crane/Logger.h + include/crane/jwt.h include/crane/PasswordEntry.h include/crane/AtomicHashMap.h GrpcHelper.cpp @@ -16,6 +17,7 @@ target_link_libraries(Utility_PublicHeader PUBLIC spdlog::spdlog crane_proto_lib fpm + jwt-cpp::jwt-cpp ) # This trimmed version is used for PAM module diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index f70653300..3ed97bf7f 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -19,6 +19,8 @@ #pragma once #include +#include +#include #include struct TlsCertificates { @@ -30,10 +32,27 @@ struct TlsCertificates { }; struct ClientTlsCertificates { - std::string ClientCertFilePath; - std::string ClientCertContent; + std::string ClientCertFilePath; + std::string ClientCertContent; }; +class MyAuthProcessor : public grpc::AuthMetadataProcessor { + public: + grpc::Status Process(const InputMetadata& auth_metadata, + grpc::AuthContext* context, + OutputMetadata* consumed_auth_metadata, + OutputMetadata* response_metadata) override { + for (const auto& [k, v] : auth_metadata) { + std::cout << k << " " << v << std::endl; + } + auto kv = auth_metadata.find("Authorization"); + if (strcmp(kv->second.data(), "Token") != 0) { + return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Invalid token"); + } + + return grpc::Status::OK; + } +}; void ServerBuilderSetCompression(grpc::ServerBuilder* builder); @@ -47,9 +66,10 @@ void ServerBuilderAddTcpInsecureListeningPort(grpc::ServerBuilder* builder, const std::string& port); void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, - const std::string& address, - const std::string& port, - const TlsCertificates& certs, const std::string pem_root_cert); + const std::string& address, + const std::string& port, + const TlsCertificates& certs, + const std::string pem_root_cert); void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, @@ -74,12 +94,15 @@ std::shared_ptr CreateTcpInsecureCustomChannel( std::shared_ptr CreateTcpTlsCustomChannelByIp( const std::string& ip, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, const grpc::ChannelArguments& args); + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const grpc::ChannelArguments& args); std::shared_ptr CreateTcpTlsChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, const std::string& domainSuffix); + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const std::string& domainSuffix); std::shared_ptr CreateTcpTlsCustomChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, const std::string& domainSuffix, const grpc::ChannelArguments& args); + const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const std::string& domainSuffix, const grpc::ChannelArguments& args); diff --git a/src/Utilities/PublicHeader/include/crane/Jwt.h b/src/Utilities/PublicHeader/include/crane/Jwt.h new file mode 100644 index 000000000..37aeaca4f --- /dev/null +++ b/src/Utilities/PublicHeader/include/crane/Jwt.h @@ -0,0 +1,21 @@ + + +#pragma once + +#include + +#include +#include +#include + +namespace util { + +std::string GenerateToken( + const std::string& secret, + const std::unordered_map& claims); + +bool VerifyToken(const std::string& secret, const std::string& token); + +std::string GetClaim(const std::string& key); + +} // namespace util \ No newline at end of file diff --git a/src/Utilities/PublicHeader/jwt.cpp b/src/Utilities/PublicHeader/jwt.cpp new file mode 100644 index 000000000..24c15e86f --- /dev/null +++ b/src/Utilities/PublicHeader/jwt.cpp @@ -0,0 +1,41 @@ +#include "crane/jwt.h" + +#include + +#include "jwt-cpp/jwt.h" +#include "jwt-cpp/traits/kazuho-picojson/defaults.h" + +namespace util { +std::string GenerateToken( + const std::string& secret, + const std::unordered_map& claims) { + auto creater = jwt::create().set_issuer("crane").set_type("JWS"); + for (const auto& [k, v] : claims) { + creater.set_payload_claim(k, jwt::claim(v)); + } + return creater.sign(jwt::algorithm::hs256{secret}); +} + +bool VerifyToken(const std::string& secret, const std::string& token) { + try { + auto decoded = jwt::decode(token); + jwt::verify() + .allow_algorithm(jwt::algorithm::hs256{secret}) + .with_issuer("crane") + .verify(decoded); + } catch (std::invalid_argument& a) { + return false; + } catch (jwt::error::token_verification_exception& e) { + return false; + } + + return true; +} + +std::string GetClaim(const std::string& key, const std::string& token) { + auto decoded = jwt::decode(token); + + return decoded.get_payload_claim(key).as_string(); +} + +} // namespace util \ No newline at end of file From 06071919a0f939da45e1e888f38e1763f1c83782 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 31 Oct 2024 17:13:25 +0800 Subject: [PATCH 07/62] feat: jwt config --- etc/config.yaml | 2 ++ src/CraneCtld/CraneCtld.cpp | 27 ++++++++++++++++++++------- src/CraneCtld/CtldPublicDefs.h | 3 +++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/etc/config.yaml b/etc/config.yaml index c01fc5149..c18c7e8d8 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -26,6 +26,8 @@ InternalCaFilePath: /etc/crane/internal_ca.pem DomainSuffix: crane.com +JwtCertFilePath: /etc/crane/jwt.pem + # Ctld settings # the listening address of control machine CraneCtldListenAddr: 0.0.0.0 diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index f0ed3d4ba..84d554b1e 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -138,6 +138,26 @@ void ParseConfig(int argc, char** argv) { g_config.CraneCtldMutexFilePath = g_config.CraneBaseDir + kDefaultCraneCtldMutexFile; + if (config["JwtCertFilePath"]) { + std::string jwtCertFilePath = + config["JwtCertFilePath"].as(); + + try { + g_config.JwtSecretContent = util::ReadFileIntoString(jwtCertFilePath); + } catch (const std::exception& e) { + CRANE_ERROR("Read cert file error: {}", e.what()); + std::exit(1); + } + if (g_config.JwtSecretContent.empty()) { + CRANE_ERROR( + "The file specified by JwtCertFilePath " + "is empty"); + } + } else { + CRANE_ERROR("JwtCertFilePath is empty"); + std::exit(1); + } + if (config["CraneCtldListenAddr"]) g_config.ListenConf.CraneCtldListenAddr = config["CraneCtldListenAddr"].as(); @@ -164,13 +184,6 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.CraneCtldForCforedListenPort = kCtldForCforedDefaultPort; - if (config["CraneCtldForCforedListenPort"]) - g_config.ListenConf.CraneCtldForCforedListenPort = - config["CraneCtldForCforedListenPort"].as(); - else - g_config.ListenConf.CraneCtldForCforedListenPort = - kCtldForCforedDefaultPort; - if (config["CompressedRpc"]) g_config.CompressedRpc = config["CompressedRpc"].as(); diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 06aa0fc7d..773d77601 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -18,6 +18,8 @@ #pragma once +#include + #include "CtldPreCompiledHeader.h" #include "crane/GrpcHelper.h" // Precompiled header come first! @@ -103,6 +105,7 @@ struct Config { TlsCertsConfig TlsCerts; }; CraneCtldListenConf ListenConf; + std::string JwtSecretContent; struct CranedListenConf { std::string CranedListenPort; From 1a6b4a4eb24f0193f3a2166c86267bd924a9a415 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Fri, 1 Nov 2024 15:16:54 +0800 Subject: [PATCH 08/62] feat: User adds password information. --- protos/PublicDefs.proto | 42 +++++++++++++++++--------------- src/CraneCtld/CtldGrpcServer.cpp | 9 +++++++ src/CraneCtld/CtldPublicDefs.h | 1 + src/CraneCtld/DbClient.cpp | 1 + 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/protos/PublicDefs.proto b/protos/PublicDefs.proto index 9367c909e..d6c9e5f59 100644 --- a/protos/PublicDefs.proto +++ b/protos/PublicDefs.proto @@ -141,7 +141,7 @@ message TaskToCtld { string extra_attr = 23; string cmd_line = 31; - string cwd = 32; // Current working directory + string cwd = 32; // Current working directory map env = 33; string excludes = 34; @@ -183,7 +183,7 @@ message TaskToD { // If this task is PENDING, start_time is either not set (default constructed) // or an estimated start time. // If this task is RUNNING, start_time is the actual starting time. - google.protobuf.Timestamp start_time = 5; // Currently Only used in CraneCtld + google.protobuf.Timestamp start_time = 5; // Currently Only used in CraneCtld google.protobuf.Duration time_limit = 6; string partition = 8; @@ -276,10 +276,11 @@ message TaskInfo { string craned_list = 36; } - // The time of different nodes across the whole cluster might not always be synchronized. - // If the time on the front end node is more than several seconds ahead of the CraneCtld node, - // a negative elapsed time might occur. - // To avoid this, the elapsed time of a task is calculated on the CraneCtld side. + // The time of different nodes across the whole cluster might not always be + // synchronized. If the time on the front end node is more than several + // seconds ahead of the CraneCtld node, a negative elapsed time might occur. + // To avoid this, the elapsed time of a task is calculated on the CraneCtld + // side. google.protobuf.Duration elapsed_time = 37; repeated string execution_node = 38; } @@ -434,15 +435,17 @@ message AccountInfo { bool blocked = 10; } -// Note: UserInfo DIFFERS from the `User` struct in C++ code and database representation -// and is ONLY used for communication between CraneCtld and cacctmgr command. -// If an user belongs to multiple accounts, There will be multiple `UserInfo` -// messages with `account` pointing to each account. +// Note: UserInfo DIFFERS from the `User` struct in C++ code and database +// representation +// and is ONLY used for communication between CraneCtld and cacctmgr +// command. If an user belongs to multiple accounts, There will be +// multiple `UserInfo` messages with `account` pointing to each account. // For example, if a user (uid=1) belongs to accounts `1,2,3`, -// there will be three `UserInfo` messages: (uid=1, account=1), (uid=1, account=2), -// (uid=1, account=3). -// The c++ code and database representation use a Map to contain -// in ONE UserInfo message all the information belonging to different accounts. +// there will be three `UserInfo` messages: (uid=1, account=1), (uid=1, +// account=2), (uid=1, account=3). The c++ code and database +// representation use a Map to contain in +// ONE UserInfo message all the information belonging to different +// accounts. message UserInfo { enum AdminLevel { None = 0; @@ -458,11 +461,12 @@ message UserInfo { uint32 uid = 1; string name = 2; - string account = 3; - bool blocked = 4; - repeated AllowedPartitionQos allowed_partition_qos_list = 5; - repeated string coordinator_accounts = 6; - AdminLevel admin_level = 7; + string password = 3; + string account = 4; + bool blocked = 5; + repeated AllowedPartitionQos allowed_partition_qos_list = 6; + repeated string coordinator_accounts = 7; + AdminLevel admin_level = 8; } message QosInfo { diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index 794380a8a..e5bfd1a59 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -295,6 +295,7 @@ grpc::Status CraneCtldServiceImpl::AddUser( const crane::grpc::UserInfo *user_info = &request->user(); user.name = user_info->name(); + user.password = user_info->password(); user.uid = user_info->uid(); user.default_account = user_info->account(); user.admin_level = User::AdminLevel(user_info->admin_level()); @@ -692,6 +693,14 @@ CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { listen_conf.CraneCtldListenPort); } + // std::vector< + // std::unique_ptr> + // creators; + // creators.push_back( + // std::unique_ptr( + // new AuthInterceptorFactory())); + + // builder.experimental().SetInterceptorCreators(std::move(creators)); builder.RegisterService(m_service_impl_.get()); m_server_ = builder.BuildAndStart(); diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 773d77601..049874f37 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -724,6 +724,7 @@ struct User { bool deleted = false; uid_t uid; std::string name; + std::string password; std::string default_account; AccountToAttrsMap account_to_attrs_map; std::list coordinator_accounts; diff --git a/src/CraneCtld/DbClient.cpp b/src/CraneCtld/DbClient.cpp index 03ee66eb6..f0c319d49 100644 --- a/src/CraneCtld/DbClient.cpp +++ b/src/CraneCtld/DbClient.cpp @@ -95,6 +95,7 @@ bool MongodbClient::CheckDefaultRootAccountUserAndInit_() { CRANE_TRACE("Default user ROOT not found. Insert it into DB."); root_user.name = "root"; + root_user.password = "root"; root_user.default_account = "ROOT"; root_user.admin_level = User::Root; root_user.uid = 0; From 0cc8127cb4dc814b009bf497f6e5fb1756dfc5ca Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:28:47 +0800 Subject: [PATCH 09/62] refactor: jwt-cpp uses in-project json dependency --- dependencies/cmake/CMakeLists.txt | 1 - dependencies/cmake/jwt-cpp/CMakeLists.txt | 19 +++++++++++++++++++ .../cmake/nlohmann_json/CMakeLists.txt | 19 ------------------- 3 files changed, 19 insertions(+), 20 deletions(-) delete mode 100644 dependencies/cmake/nlohmann_json/CMakeLists.txt diff --git a/dependencies/cmake/CMakeLists.txt b/dependencies/cmake/CMakeLists.txt index bc69f9a11..188f76f95 100644 --- a/dependencies/cmake/CMakeLists.txt +++ b/dependencies/cmake/CMakeLists.txt @@ -13,7 +13,6 @@ if(CRANE_USE_MIMALLOC) add_subdirectory(mimalloc) endif() add_subdirectory(BSThreadPool) -# add_subdirectory(nlohmann_json) add_subdirectory(jwt-cpp) add_subdirectory(yaml-cpp) add_subdirectory(fmt) diff --git a/dependencies/cmake/jwt-cpp/CMakeLists.txt b/dependencies/cmake/jwt-cpp/CMakeLists.txt index cc2afc5ff..058e88739 100644 --- a/dependencies/cmake/jwt-cpp/CMakeLists.txt +++ b/dependencies/cmake/jwt-cpp/CMakeLists.txt @@ -1,8 +1,27 @@ include(FetchContent) +if (CRANE_USE_GITEE_SOURCE) + set(JSON_SRC_URL "https://gitee.com/zenglingbo/crane-sched-deps/raw/master/json-3.11.2.tar.gz") +else () + set(JSON_SRC_URL "https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.tar.gz") +endif () set(JWT_CPP_SRC_URL "https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz") +FetchContent_Declare(json + URL ${JSON_SRC_URL} + URL_HASH SHA256=d69f9deb6a75e2580465c6c4c5111b89c4dc2fa94e3a85fcd2ffcd9a143d9273 + INACTIVITY_TIMEOUT 5 + ) + +FetchContent_GetProperties(json) +if (NOT json_POPULATED) + FetchContent_Populate(json) + add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR} EXCLUDE_FROM_ALL) +endif () + +set(nlohmann_json_DIR "${json_BINARY_DIR}") + fetchcontent_declare(jwt-cpp URL ${JWT_CPP_SRC_URL} URL_HASH SHA256=b9eb270e3ba8221e4b2bc38723c9a1cb4fa6c241a42908b9a334daff31137406 diff --git a/dependencies/cmake/nlohmann_json/CMakeLists.txt b/dependencies/cmake/nlohmann_json/CMakeLists.txt deleted file mode 100644 index 853347711..000000000 --- a/dependencies/cmake/nlohmann_json/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -include(FetchContent) - -if (CRANE_USE_GITEE_SOURCE) - set(JSON_SRC_URL "https://gitee.com/zenglingbo/crane-sched-deps/raw/master/json-3.11.2.tar.gz") -else () - set(JSON_SRC_URL "https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.tar.gz") -endif () - -FetchContent_Declare(json - URL ${JSON_SRC_URL} - URL_HASH SHA256=d69f9deb6a75e2580465c6c4c5111b89c4dc2fa94e3a85fcd2ffcd9a143d9273 - INACTIVITY_TIMEOUT 5 - ) - -FetchContent_GetProperties(json) -if (NOT json_POPULATED) - FetchContent_Populate(json) - add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR} EXCLUDE_FROM_ALL) -endif () \ No newline at end of file From ab44844348693fea908ab92f406efed2934ee832 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:37:17 +0800 Subject: [PATCH 10/62] feat: token --- protos/Crane.proto | 14 +++++ src/CraneCtld/CraneCtld.cpp | 4 +- src/CraneCtld/CtldGrpcServer.cpp | 28 ++++++---- src/CraneCtld/CtldGrpcServer.h | 4 ++ src/Utilities/PublicHeader/GrpcHelper.cpp | 52 ++++++++++++++--- .../PublicHeader/include/crane/GrpcHelper.h | 56 ++++++++++++++----- .../PublicHeader/include/crane/Jwt.h | 2 +- 7 files changed, 124 insertions(+), 36 deletions(-) diff --git a/protos/Crane.proto b/protos/Crane.proto index 0a1199e84..dbcb1eb91 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -745,6 +745,17 @@ message StreamTaskIOReply { } } +message LoginRequest { + uint32 uid = 1; + string password = 2; +} + +message LoginReply { + bool ok = 1; + string token = 2; + string reason = 3; +} + // Todo: Divide service into two parts: one for Craned and one for Crun // We need to distinguish the message sender // and have some kind of authentication @@ -776,6 +787,9 @@ service CraneCtld { /* RPCs called from cinfo */ rpc QueryClusterInfo(QueryClusterInfoRequest) returns (QueryClusterInfoReply); + /* RPCs called from clogin */ + rpc Login(LoginRequest) returns (LoginReply); + /* common RPCs */ rpc QueryTasksInfo(QueryTasksInfoRequest) returns (QueryTasksInfoReply); } diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 84d554b1e..c965bf013 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -16,8 +16,6 @@ * along with this program. If not, see . */ -#include "CtldForCforedServer.h" -#include "CtldForCranedServer.h" #include "CtldPreCompiledHeader.h" // Precompiled header comes first! @@ -33,6 +31,8 @@ #include "AccountMetaContainer.h" #include "CranedKeeper.h" #include "CranedMetaContainer.h" +#include "CtldForCforedServer.h" +#include "CtldForCranedServer.h" #include "CtldGrpcServer.h" #include "CtldPublicDefs.h" #include "DbClient.h" diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index e5bfd1a59..2177a5ffc 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -260,6 +260,12 @@ grpc::Status CraneCtldServiceImpl::QueryTasksInfo( return grpc::Status::OK; } +grpc::Status CraneCtldServiceImpl::Login( + grpc::ServerContext *context, const crane::grpc::LoginRequest *request, + crane::grpc::LoginReply *response) { + return grpc::Status::OK; +} + grpc::Status CraneCtldServiceImpl::AddAccount( grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) { @@ -685,22 +691,22 @@ CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { std::string cranectld_listen_addr = listen_conf.CraneCtldListenAddr; if (listen_conf.UseTls) { - ServerBuilderAddTcpTlsListeningPort(&builder, cranectld_listen_addr, - listen_conf.CraneCtldListenPort, - listen_conf.TlsCerts.ExternalCerts); + ServerBuilderAddTcpTlsListeningPort( + &builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort, + listen_conf.TlsCerts.ExternalCerts, g_config.JwtSecretContent); } else { ServerBuilderAddTcpInsecureListeningPort(&builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort); - } + std::vector< + std::unique_ptr> + creators; + creators.push_back( + std::unique_ptr( + new JwtAuthInterceptorFactory(g_config.JwtSecretContent))); - // std::vector< - // std::unique_ptr> - // creators; - // creators.push_back( - // std::unique_ptr( - // new AuthInterceptorFactory())); + builder.experimental().SetInterceptorCreators(std::move(creators)); + } - // builder.experimental().SetInterceptorCreators(std::move(creators)); builder.RegisterService(m_service_impl_.get()); m_server_ = builder.BuildAndStart(); diff --git a/src/CraneCtld/CtldGrpcServer.h b/src/CraneCtld/CtldGrpcServer.h index 631b356aa..96312dd10 100644 --- a/src/CraneCtld/CtldGrpcServer.h +++ b/src/CraneCtld/CtldGrpcServer.h @@ -76,6 +76,10 @@ class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { const crane::grpc::ModifyCranedStateRequest *request, crane::grpc::ModifyCranedStateReply *response) override; + grpc::Status Login(grpc::ServerContext *context, + const crane::grpc::LoginRequest *request, + crane::grpc::LoginReply *response) override; + grpc::Status AddAccount(grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) override; diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index c95521101..7f7e1af10 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -18,7 +18,49 @@ #include "crane/GrpcHelper.h" -#include "crane/Network.h" +// grpc::Status JwtAuthProcessor::Process(const InputMetadata& auth_metadata, +// grpc::AuthContext* context, +// OutputMetadata* +// consumed_auth_metadata, +// OutputMetadata* response_metadata) { +// auto iter = auth_metadata.find("Authorization"); +// if (iter == auth_metadata.end()) { +// return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Miss token"); +// } + +// auto token = iter->second.data(); +// if (!util::VerifyToken(jwt_secret_, token)) { +// return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Invalid token"); +// } + +// return grpc::Status::OK; +// } + +void JwtAuthInterceptor::Intercept( + grpc::experimental::InterceptorBatchMethods* methods) { + if (methods->QueryInterceptionHookPoint( + grpc::experimental::InterceptionHookPoints:: + POST_RECV_INITIAL_METADATA)) { + auto* metadata_map = methods->GetRecvInitialMetadata(); + + auto iter = metadata_map->find("Authorization"); + if (iter == metadata_map->end()) { + info_->server_context()->TryCancel(); + return; + } + std::unordered_map clams = {{"UID", "0"}}; + std::string ptoken = util::GenerateToken(jwt_secret_, clams); + std::cout << ptoken << std::endl; + auto token = iter->second.data(); + if (!util::VerifyToken(jwt_secret_, token)) { + info_->server_context()->TryCancel(); + return; + } + info_->server_context()->AddInitialMetadata("UID", + util::GetClaim("UID", token)); + } + methods->Proceed(); +} static std::string GrpcFormatIpAddress(std::string const& addr) { // Grpc needs to use [] to wrap ipv6 address @@ -64,7 +106,8 @@ void ServerBuilderAddTcpInsecureListeningPort(grpc::ServerBuilder* builder, void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, - const TlsCertificates& certs) { + const TlsCertificates& certs, + const std::string& jwt_secret) { std::string listen_addr_port = fmt::format("{}:{}", GrpcFormatIpAddress(address), port); @@ -73,11 +116,6 @@ void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, pem_key_cert_pair.private_key = certs.ServerKeyContent; grpc::SslServerCredentialsOptions ssl_opts; - // pem_root_certs is actually the certificate of server side rather than - // CA certificate. CA certificate is not needed. - // Since we use the same cert/key pair for both cranectld/craned, - // pem_root_certs is set to the same certificate. - // ssl_opts.pem_root_certs = certs.ServerCertContent; ssl_opts.pem_key_cert_pairs.emplace_back(std::move(pem_key_cert_pair)); ssl_opts.client_certificate_request = GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE; diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index 3ed97bf7f..d3a82b5d4 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -20,9 +20,13 @@ #include #include +#include #include #include +#include "crane/Network.h" +#include "crane/jwt.h" + struct TlsCertificates { std::string ServerCertFilePath; std::string ServerCertContent; @@ -36,22 +40,43 @@ struct ClientTlsCertificates { std::string ClientCertContent; }; -class MyAuthProcessor : public grpc::AuthMetadataProcessor { +// class JwtAuthProcessor : public grpc::AuthMetadataProcessor { +// public: +// JwtAuthProcessor(std::string secret) : jwt_secret_(secret) {} +// grpc::Status Process(const InputMetadata& auth_metadata, +// grpc::AuthContext* context, +// OutputMetadata* consumed_auth_metadata, +// OutputMetadata* response_metadata) override; + +// private: +// std::string jwt_secret_; +// }; + +// class JwtAuthInterceptor : public grpc::experimental::Interceptor { +// public: +// explicit JwtAuthInterceptor(grpc::experimental::ServerRpcInfo* info, +// std::string secret) +// : info_(info), jwt_secret_(secret) {} + +// void Intercept(grpc::experimental::InterceptorBatchMethods* methods) +// override; + +// private: +// grpc::experimental::ServerRpcInfo* info_; +// std::string jwt_secret_; +// }; + +class JwtAuthInterceptorFactory + : public grpc::experimental::ServerInterceptorFactoryInterface { public: - grpc::Status Process(const InputMetadata& auth_metadata, - grpc::AuthContext* context, - OutputMetadata* consumed_auth_metadata, - OutputMetadata* response_metadata) override { - for (const auto& [k, v] : auth_metadata) { - std::cout << k << " " << v << std::endl; - } - auto kv = auth_metadata.find("Authorization"); - if (strcmp(kv->second.data(), "Token") != 0) { - return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Invalid token"); - } - - return grpc::Status::OK; + JwtAuthInterceptorFactory(std::string secret) : jwt_secret_(secret) {} + grpc::experimental::Interceptor* CreateServerInterceptor( + grpc::experimental::ServerRpcInfo* info) override { + return new JwtAuthInterceptor(info, jwt_secret_); } + + private: + std::string jwt_secret_; }; void ServerBuilderSetCompression(grpc::ServerBuilder* builder); @@ -74,7 +99,8 @@ void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, - const TlsCertificates& certs); + const TlsCertificates& certs, + const std::string& jwt_secret); void SetGrpcClientKeepAliveChannelArgs(grpc::ChannelArguments* args); diff --git a/src/Utilities/PublicHeader/include/crane/Jwt.h b/src/Utilities/PublicHeader/include/crane/Jwt.h index 37aeaca4f..5110b6980 100644 --- a/src/Utilities/PublicHeader/include/crane/Jwt.h +++ b/src/Utilities/PublicHeader/include/crane/Jwt.h @@ -16,6 +16,6 @@ std::string GenerateToken( bool VerifyToken(const std::string& secret, const std::string& token); -std::string GetClaim(const std::string& key); +std::string GetClaim(const std::string& key, const std::string& token); } // namespace util \ No newline at end of file From d4208086ee7c5238d7b1ec9b34635b304d7b706c Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:49:37 +0800 Subject: [PATCH 11/62] refactor --- src/Utilities/PublicHeader/CMakeLists.txt | 4 ++-- src/Utilities/PublicHeader/GrpcHelper.cpp | 3 --- .../PublicHeader/{jwt.cpp => Jwt.cpp} | 9 ++------ .../PublicHeader/include/crane/GrpcHelper.h | 23 +++++++++---------- .../PublicHeader/include/crane/Jwt.h | 1 - 5 files changed, 15 insertions(+), 25 deletions(-) rename src/Utilities/PublicHeader/{jwt.cpp => Jwt.cpp} (86%) diff --git a/src/Utilities/PublicHeader/CMakeLists.txt b/src/Utilities/PublicHeader/CMakeLists.txt index e6e88b32b..d62e7cfb7 100644 --- a/src/Utilities/PublicHeader/CMakeLists.txt +++ b/src/Utilities/PublicHeader/CMakeLists.txt @@ -1,5 +1,5 @@ add_library(Utility_PublicHeader - String.cpp Network.cpp OS.cpp PublicHeader.cpp Logger.cpp jwt.cpp + String.cpp Network.cpp OS.cpp PublicHeader.cpp Logger.cpp Jwt.cpp include/crane/String.h include/crane/Network.h include/crane/OS.h @@ -7,7 +7,7 @@ add_library(Utility_PublicHeader include/crane/Lock.h include/crane/Pointer.h include/crane/Logger.h - include/crane/jwt.h + include/crane/Jwt.h include/crane/PasswordEntry.h include/crane/AtomicHashMap.h GrpcHelper.cpp diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index 7f7e1af10..a8b26b050 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -48,9 +48,6 @@ void JwtAuthInterceptor::Intercept( info_->server_context()->TryCancel(); return; } - std::unordered_map clams = {{"UID", "0"}}; - std::string ptoken = util::GenerateToken(jwt_secret_, clams); - std::cout << ptoken << std::endl; auto token = iter->second.data(); if (!util::VerifyToken(jwt_secret_, token)) { info_->server_context()->TryCancel(); diff --git a/src/Utilities/PublicHeader/jwt.cpp b/src/Utilities/PublicHeader/Jwt.cpp similarity index 86% rename from src/Utilities/PublicHeader/jwt.cpp rename to src/Utilities/PublicHeader/Jwt.cpp index 24c15e86f..22ddd88fa 100644 --- a/src/Utilities/PublicHeader/jwt.cpp +++ b/src/Utilities/PublicHeader/Jwt.cpp @@ -1,9 +1,4 @@ -#include "crane/jwt.h" - -#include - -#include "jwt-cpp/jwt.h" -#include "jwt-cpp/traits/kazuho-picojson/defaults.h" +#include "crane/Jwt.h" namespace util { std::string GenerateToken( @@ -38,4 +33,4 @@ std::string GetClaim(const std::string& key, const std::string& token) { return decoded.get_payload_claim(key).as_string(); } -} // namespace util \ No newline at end of file +} // namespace util diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index d3a82b5d4..496408ad9 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -24,8 +24,8 @@ #include #include +#include "crane/Jwt.h" #include "crane/Network.h" -#include "crane/jwt.h" struct TlsCertificates { std::string ServerCertFilePath; @@ -52,19 +52,18 @@ struct ClientTlsCertificates { // std::string jwt_secret_; // }; -// class JwtAuthInterceptor : public grpc::experimental::Interceptor { -// public: -// explicit JwtAuthInterceptor(grpc::experimental::ServerRpcInfo* info, -// std::string secret) -// : info_(info), jwt_secret_(secret) {} +class JwtAuthInterceptor : public grpc::experimental::Interceptor { + public: + explicit JwtAuthInterceptor(grpc::experimental::ServerRpcInfo* info, + std::string secret) + : info_(info), jwt_secret_(secret) {} -// void Intercept(grpc::experimental::InterceptorBatchMethods* methods) -// override; + void Intercept(grpc::experimental::InterceptorBatchMethods* methods) override; -// private: -// grpc::experimental::ServerRpcInfo* info_; -// std::string jwt_secret_; -// }; + private: + grpc::experimental::ServerRpcInfo* info_; + std::string jwt_secret_; +}; class JwtAuthInterceptorFactory : public grpc::experimental::ServerInterceptorFactoryInterface { diff --git a/src/Utilities/PublicHeader/include/crane/Jwt.h b/src/Utilities/PublicHeader/include/crane/Jwt.h index 5110b6980..b298ae04b 100644 --- a/src/Utilities/PublicHeader/include/crane/Jwt.h +++ b/src/Utilities/PublicHeader/include/crane/Jwt.h @@ -4,7 +4,6 @@ #include -#include #include #include From 8b372b6f271a39abaf1766df2a6e05a59982eec2 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:43:54 +0800 Subject: [PATCH 12/62] refactor --- src/CraneCtld/CtldGrpcServer.cpp | 23 ++++++++++--------- src/Utilities/PublicHeader/GrpcHelper.cpp | 3 +-- .../PublicHeader/include/crane/GrpcHelper.h | 3 +-- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index 2177a5ffc..812048137 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -691,22 +691,23 @@ CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { std::string cranectld_listen_addr = listen_conf.CraneCtldListenAddr; if (listen_conf.UseTls) { - ServerBuilderAddTcpTlsListeningPort( - &builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort, - listen_conf.TlsCerts.ExternalCerts, g_config.JwtSecretContent); + ServerBuilderAddTcpTlsListeningPort(&builder, cranectld_listen_addr, + listen_conf.CraneCtldListenPort, + listen_conf.TlsCerts.ExternalCerts); } else { ServerBuilderAddTcpInsecureListeningPort(&builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort); - std::vector< - std::unique_ptr> - creators; - creators.push_back( - std::unique_ptr( - new JwtAuthInterceptorFactory(g_config.JwtSecretContent))); - - builder.experimental().SetInterceptorCreators(std::move(creators)); } + std::vector< + std::unique_ptr> + creators; + creators.push_back( + std::unique_ptr( + new JwtAuthInterceptorFactory(g_config.JwtSecretContent))); + + builder.experimental().SetInterceptorCreators(std::move(creators)); + builder.RegisterService(m_service_impl_.get()); m_server_ = builder.BuildAndStart(); diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index a8b26b050..d978ee69e 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -103,8 +103,7 @@ void ServerBuilderAddTcpInsecureListeningPort(grpc::ServerBuilder* builder, void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, - const TlsCertificates& certs, - const std::string& jwt_secret) { + const TlsCertificates& certs) { std::string listen_addr_port = fmt::format("{}:{}", GrpcFormatIpAddress(address), port); diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index 496408ad9..277d1f1ee 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -98,8 +98,7 @@ void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, - const TlsCertificates& certs, - const std::string& jwt_secret); + const TlsCertificates& certs); void SetGrpcClientKeepAliveChannelArgs(grpc::ChannelArguments* args); From 8c9112deda97f1b5999bcd18f0acbdc409681a1c Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:27:53 +0800 Subject: [PATCH 13/62] feat: login --- src/CraneCtld/AccountManager.cpp | 50 ++++++++++++++++++++++++++++++++ src/CraneCtld/AccountManager.h | 2 ++ src/CraneCtld/CtldGrpcServer.cpp | 9 ++++++ 3 files changed, 61 insertions(+) diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 7da2e50f7..08ad19a4c 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -26,6 +26,56 @@ namespace Ctld { AccountManager::AccountManager() { InitDataMap_(); } +AccountManager::Result AccountManager::Login(uint32_t uid, + const std::string& password) { + util::read_lock_guard user_guard(m_rw_user_mutex_); + + PasswordEntry entry(uid); + if (!entry.Valid()) { + return Result{false, fmt::format("Uid {} not existed", uid)}; + } + + const User* user = GetExistedUserInfoNoLock_(entry.Username()); + if (!user) { + return Result{false, "user not existed"}; + } + + if (password != user->password) { + return Result{false, "Incorrect password"}; + } + std::unordered_map claims{ + {"UID", std::to_string(uid)}}; + const std::string& token = + util::GenerateToken(g_config.JwtSecretContent, claims); + + return Result{true, token}; +} + +AccountManager::Result AccountManager::Login(uint32_t uid, + const std::string& password) { + util::read_lock_guard user_guard(m_rw_user_mutex_); + + PasswordEntry entry(uid); + if (!entry.Valid()) { + return Result{false, fmt::format("Uid {} not existed", uid)}; + } + + const User* user = GetExistedUserInfoNoLock_(entry.Username()); + if (!user) { + return Result{false, "user not existed"}; + } + + if (password != user->password) { + return Result{false, "Incorrect password"}; + } + std::unordered_map claims{ + {"UID", std::to_string(uid)}}; + const std::string& token = + util::GenerateToken(g_config.JwtSecretContent, claims); + + return Result{true, token}; +} + AccountManager::CraneExpected AccountManager::AddUser( uint32_t uid, const User& new_user) { CraneExpected result; diff --git a/src/CraneCtld/AccountManager.h b/src/CraneCtld/AccountManager.h index 467e5ffbd..0d82aa6d0 100644 --- a/src/CraneCtld/AccountManager.h +++ b/src/CraneCtld/AccountManager.h @@ -49,6 +49,8 @@ class AccountManager { ~AccountManager() = default; + Result Login(uint32_t uid, const std::string& password); + CraneExpected AddUser(uint32_t uid, const User& new_user); CraneExpected AddAccount(uint32_t uid, const Account& new_account); diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index 812048137..06df561e0 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -263,6 +263,15 @@ grpc::Status CraneCtldServiceImpl::QueryTasksInfo( grpc::Status CraneCtldServiceImpl::Login( grpc::ServerContext *context, const crane::grpc::LoginRequest *request, crane::grpc::LoginReply *response) { + AccountManager::Result result = + g_account_manager->Login(request->uid(), request->password()); + + response->set_ok(result.ok); + if (result.ok) { + response->set_token(response->reason()); + } else { + response->set_reason(response->reason()); + } return grpc::Status::OK; } From 3dc9387ab45e6799cbfed4c315fafdccd40d0895 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:04:47 +0800 Subject: [PATCH 14/62] feat: login allowlist --- src/CraneCtld/AccountManager.cpp | 1 - src/CraneCtld/CtldGrpcServer.cpp | 4 ++-- src/Utilities/PublicHeader/GrpcHelper.cpp | 28 ++++++++++++----------- src/Utilities/PublicHeader/Jwt.cpp | 2 ++ 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 08ad19a4c..479b842be 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -39,7 +39,6 @@ AccountManager::Result AccountManager::Login(uint32_t uid, if (!user) { return Result{false, "user not existed"}; } - if (password != user->password) { return Result{false, "Incorrect password"}; } diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index 06df561e0..e60a15c12 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -268,9 +268,9 @@ grpc::Status CraneCtldServiceImpl::Login( response->set_ok(result.ok); if (result.ok) { - response->set_token(response->reason()); + response->set_token(result.reason); } else { - response->set_reason(response->reason()); + response->set_reason(result.reason); } return grpc::Status::OK; } diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index d978ee69e..2d0a17457 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -41,20 +41,22 @@ void JwtAuthInterceptor::Intercept( if (methods->QueryInterceptionHookPoint( grpc::experimental::InterceptionHookPoints:: POST_RECV_INITIAL_METADATA)) { - auto* metadata_map = methods->GetRecvInitialMetadata(); - - auto iter = metadata_map->find("Authorization"); - if (iter == metadata_map->end()) { - info_->server_context()->TryCancel(); - return; - } - auto token = iter->second.data(); - if (!util::VerifyToken(jwt_secret_, token)) { - info_->server_context()->TryCancel(); - return; + if (strcmp(info_->method(), "/crane.grpc.CraneCtld/Login") != 0) { + auto* metadata_map = methods->GetRecvInitialMetadata(); + + auto iter = metadata_map->find("authorization"); + if (iter == metadata_map->end()) { + info_->server_context()->TryCancel(); + return; + } + auto token = iter->second.data(); + if (!util::VerifyToken(jwt_secret_, token)) { + info_->server_context()->TryCancel(); + return; + } + info_->server_context()->AddInitialMetadata("uid", + util::GetClaim("UID", token)); } - info_->server_context()->AddInitialMetadata("UID", - util::GetClaim("UID", token)); } methods->Proceed(); } diff --git a/src/Utilities/PublicHeader/Jwt.cpp b/src/Utilities/PublicHeader/Jwt.cpp index 22ddd88fa..78e3d4810 100644 --- a/src/Utilities/PublicHeader/Jwt.cpp +++ b/src/Utilities/PublicHeader/Jwt.cpp @@ -20,6 +20,8 @@ bool VerifyToken(const std::string& secret, const std::string& token) { .verify(decoded); } catch (std::invalid_argument& a) { return false; + } catch (std::runtime_error& e) { + return false; } catch (jwt::error::token_verification_exception& e) { return false; } From 6b8653b711e1ba2f03f04934bccb4c9b32094a07 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:48:08 +0800 Subject: [PATCH 15/62] fix: The backend retrieves the token with control characters present. --- src/CraneCtld/CraneCtld.cpp | 5 ----- src/Utilities/PublicHeader/GrpcHelper.cpp | 4 +++- src/Utilities/PublicHeader/Jwt.cpp | 2 -- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index c965bf013..200132fdf 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -24,8 +24,6 @@ #include #include -#include -#include #include "AccountManager.h" #include "AccountMetaContainer.h" @@ -38,11 +36,8 @@ #include "DbClient.h" #include "EmbeddedDbClient.h" #include "TaskScheduler.h" -#include "crane/GrpcHelper.h" -#include "crane/Logger.h" #include "crane/Network.h" #include "crane/PluginClient.h" -#include "crane/PublicHeader.h" void ParseConfig(int argc, char** argv) { cxxopts::Options options("cranectld"); diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index 2d0a17457..a3fbb8244 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -49,7 +49,9 @@ void JwtAuthInterceptor::Intercept( info_->server_context()->TryCancel(); return; } - auto token = iter->second.data(); + char token[iter->second.size()]; + std::strcpy(token, iter->second.data()); + token[iter->second.size()] = '\0'; if (!util::VerifyToken(jwt_secret_, token)) { info_->server_context()->TryCancel(); return; diff --git a/src/Utilities/PublicHeader/Jwt.cpp b/src/Utilities/PublicHeader/Jwt.cpp index 78e3d4810..de217dc49 100644 --- a/src/Utilities/PublicHeader/Jwt.cpp +++ b/src/Utilities/PublicHeader/Jwt.cpp @@ -22,8 +22,6 @@ bool VerifyToken(const std::string& secret, const std::string& token) { return false; } catch (std::runtime_error& e) { return false; - } catch (jwt::error::token_verification_exception& e) { - return false; } return true; From 9c20e438c6dacc7e7767fa077bf72d9bd67361a8 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:03:37 +0800 Subject: [PATCH 16/62] refactor: config --- etc/config.yaml | 29 ++++++----- src/CraneCtld/AccountManager.cpp | 2 +- src/CraneCtld/CraneCtld.cpp | 39 +++++++------- src/CraneCtld/CranedKeeper.cpp | 2 - src/CraneCtld/CtldForCforedServer.cpp | 24 ++++----- src/CraneCtld/CtldForCforedServer.h | 75 +++++++++++++++------------ src/CraneCtld/CtldGrpcServer.cpp | 2 +- src/CraneCtld/CtldPublicDefs.h | 7 ++- src/Craned/Craned.cpp | 28 +++++----- 9 files changed, 108 insertions(+), 100 deletions(-) diff --git a/etc/config.yaml b/etc/config.yaml index c18c7e8d8..41dd1f849 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -12,20 +12,21 @@ CraneBaseDir: /var/crane/ # Tls settings UseTls: true -CranectldExternalCertFilePath: /etc/crane/cranectld_external.pem -CranectldExternalKeyFilePath: /etc/crane/cranectld_external.key -ExternalCaFilePath: /etc/crane/external_ca.pem - -CranectldInternalCertFilePath: /etc/crane/cranectld_internal.pem -CranectldInternalKeyFilePath: /etc/crane/cranectld_internal.key -CranedCertFilePath: /etc/crane/craned.pem -CranedKeyFilePath: /etc/crane/craned.key -CforedCertFilePath: /etc/crane/cfored.pem -CforedKeyFilePath: /etc/crane/cfored.key -InternalCaFilePath: /etc/crane/internal_ca.pem - -DomainSuffix: crane.com - +SSL: + CranectldExternalCertFilePath: /etc/crane/cranectld_external.pem + CranectldExternalKeyFilePath: /etc/crane/cranectld_external.key + ExternalCaFilePath: /etc/crane/external_ca.pem + + CranectldInternalCertFilePath: /etc/crane/cranectld_internal.pem + CranectldInternalKeyFilePath: /etc/crane/cranectld_internal.key + CranedCertFilePath: /etc/crane/craned.pem + CranedKeyFilePath: /etc/crane/craned.key + CforedCertFilePath: /etc/crane/cfored.pem + CforedKeyFilePath: /etc/crane/cfored.key + InternalCaFilePath: /etc/crane/internal_ca.pem + + DomainSuffix: crane.com + JwtCertFilePath: /etc/crane/jwt.pem # Ctld settings diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 479b842be..3b0371a6c 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -45,7 +45,7 @@ AccountManager::Result AccountManager::Login(uint32_t uid, std::unordered_map claims{ {"UID", std::to_string(uid)}}; const std::string& token = - util::GenerateToken(g_config.JwtSecretContent, claims); + util::GenerateToken(g_config.ListenConf.JwtSecretContent, claims); return Result{true, token}; } diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 200132fdf..a93a23aa8 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -138,12 +138,13 @@ void ParseConfig(int argc, char** argv) { config["JwtCertFilePath"].as(); try { - g_config.JwtSecretContent = util::ReadFileIntoString(jwtCertFilePath); + g_config.ListenConf.JwtSecretContent = + util::ReadFileIntoString(jwtCertFilePath); } catch (const std::exception& e) { CRANE_ERROR("Read cert file error: {}", e.what()); std::exit(1); } - if (g_config.JwtSecretContent.empty()) { + if (g_config.ListenConf.JwtSecretContent.empty()) { CRANE_ERROR( "The file specified by JwtCertFilePath " "is empty"); @@ -183,6 +184,8 @@ void ParseConfig(int argc, char** argv) { g_config.CompressedRpc = config["CompressedRpc"].as(); if (config["UseTls"] && config["UseTls"].as()) { + const auto& ssl_config = config["SSL"]; + TlsCertificates& external_certs = g_config.ListenConf.TlsCerts.ExternalCerts; TlsCertificates& internal_certs = @@ -194,13 +197,13 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.UseTls = true; - if (config["DomainSuffix"]) + if (ssl_config["DomainSuffix"]) g_config.ListenConf.TlsCerts.DomainSuffix = - config["DomainSuffix"].as(); + ssl_config["DomainSuffix"].as(); - if (config["CranectldExternalCertFilePath"]) { + if (ssl_config["CranectldExternalCertFilePath"]) { external_certs.ServerCertFilePath = - config["CranectldExternalCertFilePath"].as(); + ssl_config["CranectldExternalCertFilePath"].as(); try { external_certs.ServerCertContent = @@ -221,9 +224,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CranectldExternalKeyFilePath"]) { + if (ssl_config["CranectldExternalKeyFilePath"]) { external_certs.ServerKeyFilePath = - config["CranectldExternalKeyFilePath"].as(); + ssl_config["CranectldExternalKeyFilePath"].as(); try { external_certs.ServerKeyContent = @@ -244,9 +247,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["InternalCaFilePath"]) { + if (ssl_config["InternalCaFilePath"]) { std::string internalCaFilePath = - config["InternalCaFilePath"].as(); + ssl_config["InternalCaFilePath"].as(); try { g_config.ListenConf.TlsCerts.InternalCaContent = @@ -265,9 +268,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CranectldInternalCertFilePath"]) { + if (ssl_config["CranectldInternalCertFilePath"]) { internal_certs.ServerCertFilePath = - config["CranectldInternalCertFilePath"].as(); + ssl_config["CranectldInternalCertFilePath"].as(); try { internal_certs.ServerCertContent = @@ -288,9 +291,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CranectldInternalKeyFilePath"]) { + if (ssl_config["CranectldInternalKeyFilePath"]) { internal_certs.ServerKeyFilePath = - config["CranectldInternalKeyFilePath"].as(); + ssl_config["CranectldInternalKeyFilePath"].as(); try { internal_certs.ServerKeyContent = @@ -311,9 +314,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CranedCertFilePath"]) { + if (ssl_config["CranedCertFilePath"]) { craned_certs.ClientCertFilePath = - config["CranedCertFilePath"].as(); + ssl_config["CranedCertFilePath"].as(); try { craned_certs.ClientCertContent = @@ -332,9 +335,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CforedCertFilePath"]) { + if (ssl_config["CforedCertFilePath"]) { cfored_certs.ClientCertFilePath = - config["CforedCertFilePath"].as(); + ssl_config["CforedCertFilePath"].as(); try { cfored_certs.ClientCertContent = diff --git a/src/CraneCtld/CranedKeeper.cpp b/src/CraneCtld/CranedKeeper.cpp index 39c433e39..85f408546 100644 --- a/src/CraneCtld/CranedKeeper.cpp +++ b/src/CraneCtld/CranedKeeper.cpp @@ -20,8 +20,6 @@ #include -#include "CtldPublicDefs.h" - namespace Ctld { using grpc::ClientContext; diff --git a/src/CraneCtld/CtldForCforedServer.cpp b/src/CraneCtld/CtldForCforedServer.cpp index 450804063..400693b3f 100644 --- a/src/CraneCtld/CtldForCforedServer.cpp +++ b/src/CraneCtld/CtldForCforedServer.cpp @@ -18,13 +18,11 @@ #include #include -#include #include "AccountManager.h" #include "CranedKeeper.h" #include "CranedMetaContainer.h" #include "CtldGrpcServer.h" -#include "CtldPublicDefs.h" #include "EmbeddedDbClient.h" #include "TaskScheduler.h" #include "crane/String.h" @@ -209,22 +207,24 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( } } -CtldForCforedServer::CtldForCforedServer(const Config::CraneCtldListenConf &listen_conf) { +CtldForCforedServer::CtldForCforedServer( + const Config::CraneCtldListenConf &listen_conf) { m_service_impl_ = std::make_unique(this); grpc::ServerBuilder builder; if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); - if (listen_conf.UseTls) + if (listen_conf.UseTls) ServerBuilderAddmTcpTlsListeningPort( &builder, listen_conf.CraneCtldListenAddr, - listen_conf.CraneCtldForCforedListenPort, listen_conf.TlsCerts.InternalCerts, + listen_conf.CraneCtldForCforedListenPort, + listen_conf.TlsCerts.InternalCerts, listen_conf.TlsCerts.InternalCaContent); - else - ServerBuilderAddTcpInsecureListeningPort(&builder, - listen_conf.CraneCtldListenAddr, - listen_conf.CraneCtldForCforedListenPort); + else + ServerBuilderAddTcpInsecureListeningPort( + &builder, listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForCforedListenPort); builder.RegisterService(m_service_impl_.get()); m_server_ = builder.BuildAndStart(); @@ -233,8 +233,8 @@ CtldForCforedServer::CtldForCforedServer(const Config::CraneCtldListenConf &list std::exit(1); } CRANE_INFO("CraneCtld For Cfored Server is listening on {}:{} and Tls is {}", - listen_conf.CraneCtldListenAddr, listen_conf.CraneCtldForCforedListenPort, - listen_conf.UseTls); + listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForCforedListenPort, listen_conf.UseTls); } -} // namespace Ctld \ No newline at end of file +} // namespace Ctld \ No newline at end of file diff --git a/src/CraneCtld/CtldForCforedServer.h b/src/CraneCtld/CtldForCforedServer.h index ac3c09b77..ad1aabdee 100644 --- a/src/CraneCtld/CtldForCforedServer.h +++ b/src/CraneCtld/CtldForCforedServer.h @@ -16,8 +16,6 @@ #pragma once -#include -#include "CtldForCranedServer.h" #include "CtldPublicDefs.h" // Precompiled header comes first! @@ -26,7 +24,7 @@ #include "protos/Crane.pb.h" namespace Ctld { - + using crane::grpc::Craned; using grpc::Channel; using grpc::Server; @@ -44,9 +42,8 @@ class CforedStreamWriter { crane::grpc::StreamCforedRequest> *stream) : m_stream_(stream), m_valid_(true) {} - bool WriteTaskIdReply( - pid_t calloc_pid, - result::result res) { + bool WriteTaskIdReply(pid_t calloc_pid, + result::result res) { LockGuard guard(&m_stream_mtx_); if (!m_valid_) return false; @@ -66,8 +63,11 @@ class CforedStreamWriter { return m_stream_->Write(reply); } - bool WriteTaskResAllocReply(task_id_t task_id, - result::result>, std::string> res) { + bool WriteTaskResAllocReply( + task_id_t task_id, + result::result>, + std::string> + res) { LockGuard guard(&m_stream_mtx_); if (!m_valid_) return false; @@ -78,8 +78,12 @@ class CforedStreamWriter { if (res.has_value()) { task_res_alloc_reply->set_ok(true); - task_res_alloc_reply->set_allocated_craned_regex(std::move(res.value().first)); - std::ranges::for_each(res.value().second,[&task_res_alloc_reply](const auto& craned_id){task_res_alloc_reply->add_craned_ids(craned_id);}); + task_res_alloc_reply->set_allocated_craned_regex( + std::move(res.value().first)); + std::ranges::for_each(res.value().second, + [&task_res_alloc_reply](const auto &craned_id) { + task_res_alloc_reply->add_craned_ids(craned_id); + }); } else { task_res_alloc_reply->set_ok(false); task_res_alloc_reply->set_failure_reason(std::move(res.error())); @@ -91,7 +95,8 @@ class CforedStreamWriter { bool WriteTaskCompletionAckReply(task_id_t task_id) { LockGuard guard(&m_stream_mtx_); if (!m_valid_) return false; - CRANE_TRACE("Sending TaskCompletionAckReply to cfored of task id {}",task_id); + CRANE_TRACE("Sending TaskCompletionAckReply to cfored of task id {}", + task_id); StreamCtldReply reply; reply.set_type(StreamCtldReply::TASK_COMPLETION_ACK_REPLY); @@ -163,50 +168,52 @@ class CforedStreamWriter { class CtldForCforedServer; -class CtldForCforedServiceImpl final : public crane::grpc::CraneCtldForCfored::Service { - public: - explicit CtldForCforedServiceImpl(CtldForCforedServer *server) : m_ctld_server_(server) {} - - grpc::Status CforedStream( +class CtldForCforedServiceImpl final + : public crane::grpc::CraneCtldForCfored::Service { + public: + explicit CtldForCforedServiceImpl(CtldForCforedServer *server) + : m_ctld_server_(server) {} + + grpc::Status CforedStream( grpc::ServerContext *context, grpc::ServerReaderWriter *stream) override; - private: - CtldForCforedServer *m_ctld_server_; + private: + CtldForCforedServer *m_ctld_server_; }; /*** * Note: There should be only ONE instance of CtldServer!!!! */ class CtldForCforedServer { - public: - explicit CtldForCforedServer(const Config::CraneCtldListenConf &listen_conf); + public: + explicit CtldForCforedServer(const Config::CraneCtldListenConf &listen_conf); + + inline void Wait() { m_server_->Wait(); } - inline void Wait() { m_server_->Wait(); } - private: - template > - using HashMap = absl::flat_hash_map; + using HashMap = absl::flat_hash_map; - template > - using HashSet = absl::flat_hash_set; + using HashSet = absl::flat_hash_set; - using Mutex = util::mutex; + using Mutex = util::mutex; - std::unique_ptr m_service_impl_; - std::unique_ptr m_server_; + std::unique_ptr m_service_impl_; + std::unique_ptr m_server_; - Mutex m_mtx_; - HashMap> + Mutex m_mtx_; + HashMap> m_cfored_running_tasks_ ABSL_GUARDED_BY(m_mtx_); - friend class CtldForCforedServiceImpl; + friend class CtldForCforedServiceImpl; }; - -} // namespace Ctld +} // namespace Ctld inline std::unique_ptr g_ctld_for_cfored_server; \ No newline at end of file diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index e60a15c12..b450840ed 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -713,7 +713,7 @@ CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { creators; creators.push_back( std::unique_ptr( - new JwtAuthInterceptorFactory(g_config.JwtSecretContent))); + new JwtAuthInterceptorFactory(g_config.ListenConf.JwtSecretContent))); builder.experimental().SetInterceptorCreators(std::move(creators)); diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 049874f37..1db77dc81 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -18,12 +18,11 @@ #pragma once -#include - #include "CtldPreCompiledHeader.h" -#include "crane/GrpcHelper.h" // Precompiled header come first! +#include "crane/GrpcHelper.h" + namespace Ctld { using moodycamel::ConcurrentQueue; @@ -103,9 +102,9 @@ struct Config { }; TlsCertsConfig TlsCerts; + std::string JwtSecretContent; }; CraneCtldListenConf ListenConf; - std::string JwtSecretContent; struct CranedListenConf { std::string CranedListenPort; diff --git a/src/Craned/Craned.cpp b/src/Craned/Craned.cpp index e7175c3d2..a165f3a27 100644 --- a/src/Craned/Craned.cpp +++ b/src/Craned/Craned.cpp @@ -25,12 +25,10 @@ #include #include -#include #include "CforedClient.h" #include "CranedServer.h" #include "CtldClient.h" -#include "crane/GrpcHelper.h" #include "crane/Network.h" #include "crane/OS.h" #include "crane/PluginClient.h" @@ -173,6 +171,8 @@ void ParseConfig(int argc, char** argv) { g_config.CompressedRpc = config["CompressedRpc"].as(); if (config["UseTls"] && config["UseTls"].as()) { + const auto& ssl_config = config["SSL"]; + g_config.ListenConf.UseTls = true; TlsCertificates& craned_certs = g_config.ListenConf.TlsCerts.CranedTlsCerts; @@ -181,13 +181,13 @@ void ParseConfig(int argc, char** argv) { ClientTlsCertificates& cfoed_client_certs = g_config.ListenConf.TlsCerts.CforedClientTlsCerts; - if (config["DomainSuffix"]) + if (ssl_config["DomainSuffix"]) g_config.ListenConf.TlsCerts.DomainSuffix = - config["DomainSuffix"].as(); + ssl_config["DomainSuffix"].as(); - if (config["InternalCaFilePath"]) { + if (ssl_config["InternalCaFilePath"]) { std::string internalCaFilePath = - config["InternalCaFilePath"].as(); + ssl_config["InternalCaFilePath"].as(); try { g_config.ListenConf.TlsCerts.InternalCaContent = @@ -206,9 +206,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CranedCertFilePath"]) { + if (ssl_config["CranedCertFilePath"]) { craned_certs.ServerCertFilePath = - config["CranedCertFilePath"].as(); + ssl_config["CranedCertFilePath"].as(); try { craned_certs.ServerCertContent = @@ -227,9 +227,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CranedKeyFilePath"]) { + if (ssl_config["CranedKeyFilePath"]) { craned_certs.ServerKeyFilePath = - config["CranedKeyFilePath"].as(); + ssl_config["CranedKeyFilePath"].as(); try { craned_certs.ServerKeyContent = @@ -248,9 +248,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CranectldInternalCertFilePath"]) { + if (ssl_config["CranectldInternalCertFilePath"]) { internal_client_certs.ClientCertFilePath = - config["CranectldInternalCertFilePath"].as(); + ssl_config["CranectldInternalCertFilePath"].as(); try { internal_client_certs.ClientCertContent = util::ReadFileIntoString( @@ -271,9 +271,9 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (config["CforedCertFilePath"]) { + if (ssl_config["CforedCertFilePath"]) { cfoed_client_certs.ClientCertFilePath = - config["CforedCertFilePath"].as(); + ssl_config["CforedCertFilePath"].as(); try { cfoed_client_certs.ClientCertContent = From 56f8ce4aa8502221510db5e268e0d53d19eebc4b Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:10:26 +0800 Subject: [PATCH 17/62] feat: grpc shutdown --- src/CraneCtld/CtldForCforedServer.cpp | 5 +++++ src/CraneCtld/CtldForCforedServer.h | 2 ++ src/CraneCtld/CtldForCranedServer.cpp | 25 +++++++++++++++-------- src/CraneCtld/CtldForCranedServer.h | 13 ++++++++---- src/CraneCtld/CtldGrpcServer.cpp | 3 ++- src/CraneCtld/CtldGrpcServer.h | 2 ++ src/Utilities/PublicHeader/GrpcHelper.cpp | 4 ---- 7 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/CraneCtld/CtldForCforedServer.cpp b/src/CraneCtld/CtldForCforedServer.cpp index 400693b3f..7f8b0a645 100644 --- a/src/CraneCtld/CtldForCforedServer.cpp +++ b/src/CraneCtld/CtldForCforedServer.cpp @@ -207,6 +207,11 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( } } +void CtldForCforedServer::Shutdown() { + m_server_->Shutdown(std::chrono::system_clock::now() + + std::chrono::seconds(1)); +} + CtldForCforedServer::CtldForCforedServer( const Config::CraneCtldListenConf &listen_conf) { m_service_impl_ = std::make_unique(this); diff --git a/src/CraneCtld/CtldForCforedServer.h b/src/CraneCtld/CtldForCforedServer.h index ad1aabdee..125187a87 100644 --- a/src/CraneCtld/CtldForCforedServer.h +++ b/src/CraneCtld/CtldForCforedServer.h @@ -193,6 +193,8 @@ class CtldForCforedServer { inline void Wait() { m_server_->Wait(); } + void Shutdown(); + private: template > diff --git a/src/CraneCtld/CtldForCranedServer.cpp b/src/CraneCtld/CtldForCranedServer.cpp index 7ef450e4a..e60a60525 100644 --- a/src/CraneCtld/CtldForCranedServer.cpp +++ b/src/CraneCtld/CtldForCranedServer.cpp @@ -58,22 +58,29 @@ grpc::Status CtldForCranedServiceImpl::CranedRegister( return grpc::Status::OK; } -CtldForCranedServer::CtldForCranedServer(const Config::CraneCtldListenConf &listen_conf) { +void CtldForCranedServer::Shutdown() { + m_server_->Shutdown(std::chrono::system_clock::now() + + std::chrono::seconds(1)); +} + +CtldForCranedServer::CtldForCranedServer( + const Config::CraneCtldListenConf &listen_conf) { m_service_impl_ = std::make_unique(this); grpc::ServerBuilder builder; if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); - if (listen_conf.UseTls) + if (listen_conf.UseTls) ServerBuilderAddmTcpTlsListeningPort( &builder, listen_conf.CraneCtldListenAddr, - listen_conf.CraneCtldForCranedListenPort, listen_conf.TlsCerts.InternalCerts, + listen_conf.CraneCtldForCranedListenPort, + listen_conf.TlsCerts.InternalCerts, listen_conf.TlsCerts.InternalCaContent); - else - ServerBuilderAddTcpInsecureListeningPort(&builder, - listen_conf.CraneCtldListenAddr, - listen_conf.CraneCtldForCranedListenPort); + else + ServerBuilderAddTcpInsecureListeningPort( + &builder, listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForCranedListenPort); builder.RegisterService(m_service_impl_.get()); m_server_ = builder.BuildAndStart(); @@ -82,8 +89,8 @@ CtldForCranedServer::CtldForCranedServer(const Config::CraneCtldListenConf &list std::exit(1); } CRANE_INFO("CraneCtld For Craned Server is listening on {}:{} and Tls is {}", - listen_conf.CraneCtldListenAddr, listen_conf.CraneCtldForCranedListenPort, - listen_conf.UseTls); + listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForCranedListenPort, listen_conf.UseTls); } } // namespace Ctld \ No newline at end of file diff --git a/src/CraneCtld/CtldForCranedServer.h b/src/CraneCtld/CtldForCranedServer.h index 74f89f7e7..4c12148a7 100644 --- a/src/CraneCtld/CtldForCranedServer.h +++ b/src/CraneCtld/CtldForCranedServer.h @@ -14,7 +14,7 @@ * See the Mulan PSL v2 for more details. */ - #pragma once +#pragma once #include "CtldPublicDefs.h" // Precompiled header comes first! @@ -30,9 +30,11 @@ using grpc::Server; class CtldForCranedServer; -class CtldForCranedServiceImpl final : public crane::grpc::CraneCtldForCraned::Service { +class CtldForCranedServiceImpl final + : public crane::grpc::CraneCtldForCraned::Service { public: - explicit CtldForCranedServiceImpl(CtldForCranedServer* server) : m_ctld_for_craned_server_(server) {} + explicit CtldForCranedServiceImpl(CtldForCranedServer *server) + : m_ctld_for_craned_server_(server) {} grpc::Status TaskStatusChange( grpc::ServerContext *context, const crane::grpc::TaskStatusChangeRequest *request, @@ -42,6 +44,7 @@ class CtldForCranedServiceImpl final : public crane::grpc::CraneCtldForCraned::S grpc::ServerContext *context, const crane::grpc::CranedRegisterRequest *request, crane::grpc::CranedRegisterReply *response) override; + private: CtldForCranedServer *m_ctld_for_craned_server_; }; @@ -56,6 +59,8 @@ class CtldForCranedServer { inline void Wait() { m_server_->Wait(); } + void Shutdown(); + private: std::unique_ptr m_service_impl_; std::unique_ptr m_server_; @@ -64,5 +69,5 @@ class CtldForCranedServer { }; } // namespace Ctld - + inline std::unique_ptr g_ctld_for_craned_server; \ No newline at end of file diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/CtldGrpcServer.cpp index b450840ed..231a9bab6 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/CtldGrpcServer.cpp @@ -744,7 +744,8 @@ CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { // main thread will access g_craned_keeper simultaneously and a race // condition will occur. g_craned_keeper->Shutdown(); - + g_ctld_for_cfored_server->Shutdown(); + g_ctld_for_craned_server->Shutdown(); p_server->Shutdown(std::chrono::system_clock::now() + std::chrono::seconds(1)); }); diff --git a/src/CraneCtld/CtldGrpcServer.h b/src/CraneCtld/CtldGrpcServer.h index 96312dd10..b3293cbd3 100644 --- a/src/CraneCtld/CtldGrpcServer.h +++ b/src/CraneCtld/CtldGrpcServer.h @@ -21,6 +21,8 @@ #include "CtldPublicDefs.h" // Precompiled header comes first! +#include "CtldForCforedServer.h" +#include "CtldForCranedServer.h" #include "crane/Lock.h" #include "protos/Crane.grpc.pb.h" #include "protos/Crane.pb.h" diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index a3fbb8244..10ef8169e 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -137,10 +137,6 @@ void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, pem_key_cert_pair.private_key = certs.ServerKeyContent; grpc::SslServerCredentialsOptions ssl_opts; - // pem_root_certs is actually the certificate of server side rather than - // CA certificate. CA certificate is not needed. - // Since we use the same cert/key pair for both cranectld/craned, - // pem_root_certs is set to the same certificate. ssl_opts.pem_root_certs = pem_root_cert; ssl_opts.pem_key_cert_pairs.emplace_back(std::move(pem_key_cert_pair)); ssl_opts.client_certificate_request = From 2afefbb57481d2e9c71e302394077022e631dd1e Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:35:24 +0800 Subject: [PATCH 18/62] refactor --- src/Utilities/PublicHeader/GrpcHelper.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index 10ef8169e..327500a85 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -49,9 +49,7 @@ void JwtAuthInterceptor::Intercept( info_->server_context()->TryCancel(); return; } - char token[iter->second.size()]; - std::strcpy(token, iter->second.data()); - token[iter->second.size()] = '\0'; + std::string token(iter->second.data(), iter->second.size()); if (!util::VerifyToken(jwt_secret_, token)) { info_->server_context()->TryCancel(); return; From 655ba675603df72173ae0800aec3ccef38d3ec25 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:28:19 +0800 Subject: [PATCH 19/62] feat: refactor grpc server --- src/CraneCtld/CMakeLists.txt | 12 ++--- src/CraneCtld/RpcService/CMakeLists.txt | 47 +++++++++++++++++++ .../{ => RpcService}/CranedKeeper.cpp | 0 src/CraneCtld/{ => RpcService}/CranedKeeper.h | 2 +- .../{ => RpcService}/CtldForCforedServer.cpp | 8 ++-- .../{ => RpcService}/CtldForCforedServer.h | 2 +- .../{ => RpcService}/CtldForCranedServer.cpp | 4 +- .../{ => RpcService}/CtldForCranedServer.h | 2 +- .../{ => RpcService}/CtldGrpcServer.cpp | 9 ++-- .../{ => RpcService}/CtldGrpcServer.h | 2 +- src/CraneCtld/TaskScheduler.cpp | 6 +-- 11 files changed, 68 insertions(+), 26 deletions(-) create mode 100644 src/CraneCtld/RpcService/CMakeLists.txt rename src/CraneCtld/{ => RpcService}/CranedKeeper.cpp (100%) rename src/CraneCtld/{ => RpcService}/CranedKeeper.h (99%) rename src/CraneCtld/{ => RpcService}/CtldForCforedServer.cpp (98%) rename src/CraneCtld/{ => RpcService}/CtldForCforedServer.h (99%) rename src/CraneCtld/{ => RpcService}/CtldForCranedServer.cpp (97%) rename src/CraneCtld/{ => RpcService}/CtldForCranedServer.h (98%) rename src/CraneCtld/{ => RpcService}/CtldGrpcServer.cpp (99%) rename src/CraneCtld/{ => RpcService}/CtldGrpcServer.h (99%) diff --git a/src/CraneCtld/CMakeLists.txt b/src/CraneCtld/CMakeLists.txt index f0caaca12..26560d53c 100644 --- a/src/CraneCtld/CMakeLists.txt +++ b/src/CraneCtld/CMakeLists.txt @@ -1,17 +1,12 @@ + +add_subdirectory(RpcService) + add_executable(cranectld CtldPublicDefs.h - CtldGrpcServer.h - CtldGrpcServer.cpp - CtldForCranedServer.h - CtldForCranedServer.cpp - CtldForCforedServer.h - CtldForCforedServer.cpp DbClient.h DbClient.cpp TaskScheduler.h TaskScheduler.cpp - CranedKeeper.h - CranedKeeper.cpp CranedMetaContainer.h CranedMetaContainer.cpp AccountManager.h @@ -32,6 +27,7 @@ target_link_libraries(cranectld PRIVATE spdlog::spdlog concurrentqueue + RpcService Utility_PublicHeader Utility_PluginClient diff --git a/src/CraneCtld/RpcService/CMakeLists.txt b/src/CraneCtld/RpcService/CMakeLists.txt new file mode 100644 index 000000000..57f30e599 --- /dev/null +++ b/src/CraneCtld/RpcService/CMakeLists.txt @@ -0,0 +1,47 @@ +# Create a library from the RpcService sources +add_library(RpcService + CtldGrpcServer.h + CtldGrpcServer.cpp + CtldForCranedServer.h + CtldForCranedServer.cpp + CtldForCforedServer.h + CtldForCforedServer.cpp + CranedKeeper.h + CranedKeeper.cpp +) + +target_include_directories(RpcService PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} # Include the RpcService directory +) + +target_link_libraries(RpcService PUBLIC + spdlog::spdlog + concurrentqueue + + Utility_PublicHeader + Utility_PluginClient + + dev_event_core + dev_event_pthreads + uvw + + cxxopts + Threads::Threads + + absl::btree + absl::synchronization + absl::flat_hash_map + + crane_proto_lib + + bs_thread_pool + + yaml-cpp + mongocxx_static + + range-v3::range-v3 + result + + Backward::Interface +) + diff --git a/src/CraneCtld/CranedKeeper.cpp b/src/CraneCtld/RpcService/CranedKeeper.cpp similarity index 100% rename from src/CraneCtld/CranedKeeper.cpp rename to src/CraneCtld/RpcService/CranedKeeper.cpp diff --git a/src/CraneCtld/CranedKeeper.h b/src/CraneCtld/RpcService/CranedKeeper.h similarity index 99% rename from src/CraneCtld/CranedKeeper.h rename to src/CraneCtld/RpcService/CranedKeeper.h index 4f6f43659..b25f03125 100644 --- a/src/CraneCtld/CranedKeeper.h +++ b/src/CraneCtld/RpcService/CranedKeeper.h @@ -18,7 +18,7 @@ #pragma once -#include "CtldPublicDefs.h" +#include "../CtldPublicDefs.h" // Precompiled header comes first! #include "crane/Lock.h" diff --git a/src/CraneCtld/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp similarity index 98% rename from src/CraneCtld/CtldForCforedServer.cpp rename to src/CraneCtld/RpcService/CtldForCforedServer.cpp index 7f8b0a645..b52155e1e 100644 --- a/src/CraneCtld/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -19,12 +19,12 @@ #include #include -#include "AccountManager.h" +#include "../AccountManager.h" +#include "../CranedMetaContainer.h" +#include "../EmbeddedDbClient.h" +#include "../TaskScheduler.h" #include "CranedKeeper.h" -#include "CranedMetaContainer.h" #include "CtldGrpcServer.h" -#include "EmbeddedDbClient.h" -#include "TaskScheduler.h" #include "crane/String.h" namespace Ctld { diff --git a/src/CraneCtld/CtldForCforedServer.h b/src/CraneCtld/RpcService/CtldForCforedServer.h similarity index 99% rename from src/CraneCtld/CtldForCforedServer.h rename to src/CraneCtld/RpcService/CtldForCforedServer.h index 125187a87..ab9e6bb39 100644 --- a/src/CraneCtld/CtldForCforedServer.h +++ b/src/CraneCtld/RpcService/CtldForCforedServer.h @@ -16,7 +16,7 @@ #pragma once -#include "CtldPublicDefs.h" +#include "../CtldPublicDefs.h" // Precompiled header comes first! #include "crane/Lock.h" diff --git a/src/CraneCtld/CtldForCranedServer.cpp b/src/CraneCtld/RpcService/CtldForCranedServer.cpp similarity index 97% rename from src/CraneCtld/CtldForCranedServer.cpp rename to src/CraneCtld/RpcService/CtldForCranedServer.cpp index e60a60525..39563c2be 100644 --- a/src/CraneCtld/CtldForCranedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCranedServer.cpp @@ -18,9 +18,9 @@ #include +#include "../CranedMetaContainer.h" +#include "../TaskScheduler.h" #include "CranedKeeper.h" -#include "CranedMetaContainer.h" -#include "TaskScheduler.h" namespace Ctld { diff --git a/src/CraneCtld/CtldForCranedServer.h b/src/CraneCtld/RpcService/CtldForCranedServer.h similarity index 98% rename from src/CraneCtld/CtldForCranedServer.h rename to src/CraneCtld/RpcService/CtldForCranedServer.h index 4c12148a7..2966c8a3e 100644 --- a/src/CraneCtld/CtldForCranedServer.h +++ b/src/CraneCtld/RpcService/CtldForCranedServer.h @@ -16,7 +16,7 @@ #pragma once -#include "CtldPublicDefs.h" +#include "../CtldPublicDefs.h" // Precompiled header comes first! #include "protos/Crane.grpc.pb.h" diff --git a/src/CraneCtld/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp similarity index 99% rename from src/CraneCtld/CtldGrpcServer.cpp rename to src/CraneCtld/RpcService/CtldGrpcServer.cpp index 231a9bab6..bfe9e4933 100644 --- a/src/CraneCtld/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -18,11 +18,12 @@ #include "CtldGrpcServer.h" -#include "AccountManager.h" +#include "../AccountManager.h" +#include "../CranedMetaContainer.h" +#include "../EmbeddedDbClient.h" +#include "../TaskScheduler.h" #include "CranedKeeper.h" -#include "CranedMetaContainer.h" -#include "EmbeddedDbClient.h" -#include "TaskScheduler.h" +#include "crane/String.h" namespace Ctld { diff --git a/src/CraneCtld/CtldGrpcServer.h b/src/CraneCtld/RpcService/CtldGrpcServer.h similarity index 99% rename from src/CraneCtld/CtldGrpcServer.h rename to src/CraneCtld/RpcService/CtldGrpcServer.h index b3293cbd3..b5c38242c 100644 --- a/src/CraneCtld/CtldGrpcServer.h +++ b/src/CraneCtld/RpcService/CtldGrpcServer.h @@ -18,7 +18,7 @@ #pragma once -#include "CtldPublicDefs.h" +#include "../CtldPublicDefs.h" // Precompiled header comes first! #include "CtldForCforedServer.h" diff --git a/src/CraneCtld/TaskScheduler.cpp b/src/CraneCtld/TaskScheduler.cpp index e7e0a614f..0f16a68d9 100644 --- a/src/CraneCtld/TaskScheduler.cpp +++ b/src/CraneCtld/TaskScheduler.cpp @@ -1086,13 +1086,11 @@ TaskScheduler::SubmitTaskToScheduler(std::unique_ptr task) { err = AcquireTaskAttributes(task.get()); - if (err == CraneErr::kOk) - err = CheckTaskValidity(task.get()); + if (err == CraneErr::kOk) err = CheckTaskValidity(task.get()); if (err == CraneErr::kOk) { task->SetSubmitTime(absl::Now()); - std::future future = - SubmitTaskAsync(std::move(task)); + std::future future = SubmitTaskAsync(std::move(task)); return {std::move(future)}; } From 2f1c5d003d817479b3f610a52565bb72b12cd01f Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:43:47 +0800 Subject: [PATCH 20/62] refactor: login --- protos/Crane.proto | 16 ++++-- protos/PublicDefs.proto | 61 +++++++++++---------- src/CraneCtld/AccountManager.cpp | 44 +++------------ src/CraneCtld/AccountManager.h | 2 +- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 12 ++-- src/CraneCtld/TaskScheduler.cpp | 4 +- 6 files changed, 59 insertions(+), 80 deletions(-) diff --git a/protos/Crane.proto b/protos/Crane.proto index dbcb1eb91..f9c832453 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -753,7 +753,7 @@ message LoginRequest { message LoginReply { bool ok = 1; string token = 2; - string reason = 3; + ErrCode reason = 3; } // Todo: Divide service into two parts: one for Craned and one for Crun @@ -778,10 +778,18 @@ service CraneCtld { rpc AddUser(AddUserRequest) returns (AddUserReply); rpc AddQos(AddQosRequest) returns (AddQosReply); - rpc DeleteEntity(DeleteEntityRequest) returns (DeleteEntityReply); + rpc DeleteAccount(DeleteAccountRequest) returns (DeleteAccountReply); + rpc DeleteUser(DeleteUserRequest) returns (DeleteUserReply); + rpc DeleteQos(DeleteQosRequest) returns (DeleteQosReply); - rpc QueryEntityInfo(QueryEntityInfoRequest) returns (QueryEntityInfoReply); - rpc ModifyEntity(ModifyEntityRequest) returns (ModifyEntityReply); + rpc QueryAccountInfo(QueryAccountInfoRequest) returns (QueryAccountInfoReply); + rpc QueryUserInfo(QueryUserInfoRequest) returns (QueryUserInfoReply); + rpc QueryQosInfo(QueryQosInfoRequest) returns (QueryQosInfoReply); + + rpc ModifyAccount(ModifyAccountRequest) returns (ModifyAccountReply); + rpc ModifyUser(ModifyUserRequest) returns (ModifyUserReply); + rpc ModifyQos(ModifyQosRequest) returns (ModifyQosReply); + rpc BlockAccountOrUser(BlockAccountOrUserRequest) returns (BlockAccountOrUserReply); /* RPCs called from cinfo */ diff --git a/protos/PublicDefs.proto b/protos/PublicDefs.proto index d6c9e5f59..a06a003c2 100644 --- a/protos/PublicDefs.proto +++ b/protos/PublicDefs.proto @@ -342,36 +342,37 @@ enum ErrCode { ERR_INVALID_ADMIN_LEVEL = 10007; // Invalid permission level ERR_USER_ACCOUNT_MISMATCH = 10008; // User does not belong to the account ERR_NO_ACCOUNT_SPECIFIED = 10009; - - ERR_INVALID_ACCOUNT = 10010; // Invalid account - ERR_DUPLICATE_ACCOUNT = 10011; // Duplicate account insertion - ERR_INVALID_PARENTACCOUNT = 10012; // Invalid parent account - ERR_DELETE_ACCOUNT = 10013; // Account has child nodes - - ERR_INVALID_PARTITION = 10014; // Invalid partition, partition does not exist - ERR_ALLOWED_PARTITION = 10015; // Account/user does not include this partition - ERR_DUPLICATE_PARTITION = 10016; // Account/user duplicate insertion - ERR_PARENT_ALLOWED_PARTITION = 10017; // Parent account does not include this partition - ERR_USER_EMPTY_PARTITION = 10018; // Cannot add QoS when user has no partition - ERR_CHILD_HAS_PARTITION = 10019; // Partition '{}' is used by some descendant node of the account '{}'. Ignoring this constraint with forced operation. - - ERR_INVALID_QOS = 10020; // Invalid QoS, QoS does not exist - ERR_DB_DUPLICATE_QOS = 10021; // Duplicate QoS insertion in the database. - ERR_DELETE_QOS = 10022; // QoS reference count is not zero. - ERR_CONVERT_TO_INTERGER = 10023; // String to integer conversion failed - ERR_TIME_LIMIT = 10024; // Invalid time value - ERR_ALLOWED_QOS = 10025; // Account/user does not include this QoS. - ERR_DUPLICATE_QOS = 10026; // Account/user duplicate insertion. - ERR_PARENT_ALLOWED_QOS = 10027; // Parent account does not include this QoS. - ERR_SET_ALLOWED_QOS = 10028; // QoS '{}' is the default QoS of partition '{}', but not found in the new QoS list. - ERR_ALLOWED_DEFAULT_QOS = 10029; // Default QoS is not in the allowed QoS list - ERR_DUPLICATE_DEFAULT_QOS = 10030; // Duplicate default QoS setting - ERR_CHILD_HAS_DEFAULT_QOS = 10031; // Someone is using QoS '{}' as default QoS. Ignoring this constraint with forced deletion, the deleted default QoS is randomly replaced with one of the remaining items in the QoS list. - ERR_SET_ACCOUNT_QOS = 10032; // QoS '{}' is used by some descendant node or itself of the account '{}'. Ignoring this constraint with forced operation. - ERR_SET_DEFAULT_QOS = 10033; // Qos '{}' not in allowed qos list or is already the default qos - ERR_IS_DEFAULT_QOS = 10034; - - ERR_UPDATE_DATABASE = 10035; // Database update failed + ERR_PASSWORD_MISMATCH = 100010; + + ERR_INVALID_ACCOUNT = 10011; + ERR_DUPLICATE_ACCOUNT = 10012; + ERR_INVALID_PARENTACCOUNT = 10013; + ERR_DELETE_ACCOUNT = 10014; + + ERR_INVALID_PARTITION = 10015; + ERR_ALLOWED_PARTITION = 10016; + ERR_DUPLICATE_PARTITION = 10017; + ERR_PARENT_ALLOWED_PARTITION = 10018; + ERR_USER_EMPTY_PARTITION = 10019; + ERR_CHILD_HAS_PARTITION = 10020; + + ERR_INVALID_QOS = 10021; + ERR_DB_DUPLICATE_QOS = 10022; + ERR_DELETE_QOS = 10023; + ERR_CONVERT_TO_INTERGER = 10024; + ERR_TIME_LIMIT = 10025; + ERR_ALLOWED_QOS = 10026; + ERR_DUPLICATE_QOS = 10027; + ERR_PARENT_ALLOWED_QOS = 10028; + ERR_SET_ALLOWED_QOS = 10029; + ERR_ALLOWED_DEFAULT_QOS = 10030; + ERR_DUPLICATE_DEFAULT_QOS = 10031; + ERR_CHILD_HAS_DEFAULT_QOS = 10032; + ERR_SET_ACCOUNT_QOS = 10033; + ERR_SET_DEFAULT_QOS = 10034; + ERR_IS_DEFAULT_QOS = 10035; + + ERR_UPDATE_DATABASE = 10036; ERR_GENERIC_FAILURE = 10100; ERR_NO_RESOURCE = 10101; diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 3b0371a6c..7738d0d8d 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -26,53 +26,23 @@ namespace Ctld { AccountManager::AccountManager() { InitDataMap_(); } -AccountManager::Result AccountManager::Login(uint32_t uid, - const std::string& password) { +AccountManager::CraneExpected AccountManager::Login( + uint32_t uid, const std::string& password) { util::read_lock_guard user_guard(m_rw_user_mutex_); - PasswordEntry entry(uid); - if (!entry.Valid()) { - return Result{false, fmt::format("Uid {} not existed", uid)}; - } + auto user_result = GetUserInfoByUidNoLock_(uid); + if (!user_result) return std::unexpected(user_result.error()); + const User* user = user_result.value(); - const User* user = GetExistedUserInfoNoLock_(entry.Username()); - if (!user) { - return Result{false, "user not existed"}; - } if (password != user->password) { - return Result{false, "Incorrect password"}; + return std::unexpected(CraneErrCode::ERR_PASSWORD_MISMATCH); } std::unordered_map claims{ {"UID", std::to_string(uid)}}; const std::string& token = util::GenerateToken(g_config.ListenConf.JwtSecretContent, claims); - return Result{true, token}; -} - -AccountManager::Result AccountManager::Login(uint32_t uid, - const std::string& password) { - util::read_lock_guard user_guard(m_rw_user_mutex_); - - PasswordEntry entry(uid); - if (!entry.Valid()) { - return Result{false, fmt::format("Uid {} not existed", uid)}; - } - - const User* user = GetExistedUserInfoNoLock_(entry.Username()); - if (!user) { - return Result{false, "user not existed"}; - } - - if (password != user->password) { - return Result{false, "Incorrect password"}; - } - std::unordered_map claims{ - {"UID", std::to_string(uid)}}; - const std::string& token = - util::GenerateToken(g_config.JwtSecretContent, claims); - - return Result{true, token}; + return token; } AccountManager::CraneExpected AccountManager::AddUser( diff --git a/src/CraneCtld/AccountManager.h b/src/CraneCtld/AccountManager.h index 0d82aa6d0..a42f259fd 100644 --- a/src/CraneCtld/AccountManager.h +++ b/src/CraneCtld/AccountManager.h @@ -49,7 +49,7 @@ class AccountManager { ~AccountManager() = default; - Result Login(uint32_t uid, const std::string& password); + CraneExpected Login(uint32_t uid, const std::string& password); CraneExpected AddUser(uint32_t uid, const User& new_user); diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index bfe9e4933..1455d95bb 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -264,14 +264,14 @@ grpc::Status CraneCtldServiceImpl::QueryTasksInfo( grpc::Status CraneCtldServiceImpl::Login( grpc::ServerContext *context, const crane::grpc::LoginRequest *request, crane::grpc::LoginReply *response) { - AccountManager::Result result = - g_account_manager->Login(request->uid(), request->password()); + auto result = g_account_manager->Login(request->uid(), request->password()); - response->set_ok(result.ok); - if (result.ok) { - response->set_token(result.reason); + if (result) { + response->set_ok(true); + response->set_token(result.value()); } else { - response->set_reason(result.reason); + response->set_ok(false); + response->set_reason(result.error()); } return grpc::Status::OK; } diff --git a/src/CraneCtld/TaskScheduler.cpp b/src/CraneCtld/TaskScheduler.cpp index 0f16a68d9..c02ad7de7 100644 --- a/src/CraneCtld/TaskScheduler.cpp +++ b/src/CraneCtld/TaskScheduler.cpp @@ -1078,8 +1078,8 @@ TaskScheduler::SubmitTaskToScheduler(std::unique_ptr task) { task->Username(), task->partition_id, task->account)); } - auto enable_res = - g_account_manager->CheckEnableState(task->account, task->Username()); + auto enable_res = g_account_manager->CheckIfUserOfAccountIsEnabled( + task->Username(), task->account); if (enable_res.has_error()) { return result::fail(enable_res.error()); } From a3f6fa0f62893078e671c7700a9d56270bd15c75 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:39:40 +0800 Subject: [PATCH 21/62] refactor --- src/CraneCtld/DbClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CraneCtld/DbClient.cpp b/src/CraneCtld/DbClient.cpp index f0c319d49..f51553a1f 100644 --- a/src/CraneCtld/DbClient.cpp +++ b/src/CraneCtld/DbClient.cpp @@ -95,7 +95,7 @@ bool MongodbClient::CheckDefaultRootAccountUserAndInit_() { CRANE_TRACE("Default user ROOT not found. Insert it into DB."); root_user.name = "root"; - root_user.password = "root"; + root_user.password = ""; root_user.default_account = "ROOT"; root_user.admin_level = User::Root; root_user.uid = 0; From b0f6860ff5b2ceb9714cfaa42c916dd40da1f055 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 4 Dec 2024 11:18:08 +0800 Subject: [PATCH 22/62] feat: UID is extracted from the token. --- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 93 ++++++++++++------- src/Utilities/PublicHeader/GrpcHelper.cpp | 10 +- .../PublicHeader/include/crane/GrpcHelper.h | 2 + 3 files changed, 67 insertions(+), 38 deletions(-) diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index 1455d95bb..96341b161 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -119,9 +119,9 @@ grpc::Status CraneCtldServiceImpl::ModifyTask( grpc::ServerContext *context, const crane::grpc::ModifyTaskRequest *request, crane::grpc::ModifyTaskReply *response) { using ModifyTaskRequest = crane::grpc::ModifyTaskRequest; - - auto res = g_account_manager->CheckUidIsAdmin(request->uid()); - if (!res) { + uint32_t uid = ExtractUIDFromMetadata(context); + auto res = g_account_manager->CheckUidIsAdmin(uid); + if (res.has_error()) { for (auto task_id : request->task_ids()) { response->add_not_modified_tasks(task_id); response->add_not_modified_reasons(res.error()); @@ -279,6 +279,8 @@ grpc::Status CraneCtldServiceImpl::Login( grpc::Status CraneCtldServiceImpl::AddAccount( grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) { + uint32_t uid = ExtractUIDFromMetadata(context); + Account account; const crane::grpc::AccountInfo *account_info = &request->account(); @@ -293,7 +295,7 @@ grpc::Status CraneCtldServiceImpl::AddAccount( account.allowed_qos_list.emplace_back(qos); } - auto result = g_account_manager->AddAccount(request->uid(), account); + auto result = g_account_manager->AddAccount(uid, account); if (result) { response->set_ok(true); } else { @@ -307,6 +309,8 @@ grpc::Status CraneCtldServiceImpl::AddAccount( grpc::Status CraneCtldServiceImpl::AddUser( grpc::ServerContext *context, const crane::grpc::AddUserRequest *request, crane::grpc::AddUserReply *response) { + uint32_t uid = ExtractUIDFromMetadata(context); + User user; const crane::grpc::UserInfo *user_info = &request->user(); @@ -335,7 +339,7 @@ grpc::Status CraneCtldServiceImpl::AddUser( } AccountManager::CraneExpected result = - g_account_manager->AddUser(request->uid(), user); + g_account_manager->AddUser(uid, user); if (result) { response->set_ok(true); } else { @@ -349,6 +353,8 @@ grpc::Status CraneCtldServiceImpl::AddUser( grpc::Status CraneCtldServiceImpl::AddQos( grpc::ServerContext *context, const crane::grpc::AddQosRequest *request, crane::grpc::AddQosReply *response) { + uint32_t uid = ExtractUIDFromMetadata(context); + Qos qos; const crane::grpc::QosInfo *qos_info = &request->qos(); @@ -367,7 +373,7 @@ grpc::Status CraneCtldServiceImpl::AddQos( } qos.max_time_limit_per_task = absl::Seconds(sec); - auto result = g_account_manager->AddQos(request->uid(), qos); + auto result = g_account_manager->AddQos(uid, qos); if (result) { response->set_ok(true); } else { @@ -382,8 +388,10 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( grpc::ServerContext *context, const crane::grpc::ModifyAccountRequest *request, crane::grpc::ModifyAccountReply *response) { + uint32_t uid = ExtractUIDFromMetadata(context); + auto modify_res = g_account_manager->ModifyAccount( - request->type(), request->uid(), request->name(), request->modify_field(), + request->type(), uid, request->name(), request->modify_field(), request->value(), request->force()); if (modify_res) { @@ -399,19 +407,20 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( grpc::Status CraneCtldServiceImpl::ModifyUser( grpc::ServerContext *context, const crane::grpc::ModifyUserRequest *request, crane::grpc::ModifyUserReply *response) { + uint32_t uid = ExtractUIDFromMetadata(context); + AccountManager::CraneExpected modify_res; if (request->type() == crane::grpc::OperationType::Delete) { switch (request->modify_field()) { case crane::grpc::ModifyField::Partition: modify_res = g_account_manager->DeleteUserAllowedPartition( - request->uid(), request->name(), request->account(), - request->value()); + uid, request->name(), request->account(), request->value()); break; case crane::grpc::ModifyField::Qos: modify_res = g_account_manager->DeleteUserAllowedQos( - request->uid(), request->name(), request->partition(), - request->account(), request->value(), request->force()); + uid, request->name(), request->partition(), request->account(), + request->value(), request->force()); break; default: std::unreachable(); @@ -419,24 +428,23 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( } else { switch (request->modify_field()) { case crane::grpc::ModifyField::AdminLevel: - modify_res = g_account_manager->ModifyAdminLevel( - request->uid(), request->name(), request->value()); + modify_res = g_account_manager->ModifyAdminLevel(uid, request->name(), + request->value()); break; case crane::grpc::ModifyField::Partition: modify_res = g_account_manager->ModifyUserAllowedPartition( - request->type(), request->uid(), request->name(), request->account(), + request->type(), uid, request->name(), request->account(), request->value()); break; case crane::grpc::ModifyField::Qos: modify_res = g_account_manager->ModifyUserAllowedQos( - request->type(), request->uid(), request->name(), - request->partition(), request->account(), request->value(), - request->force()); + request->type(), uid, request->name(), request->partition(), + request->account(), request->value(), request->force()); break; case crane::grpc::ModifyField::DefaultQos: modify_res = g_account_manager->ModifyUserDefaultQos( - request->uid(), request->name(), request->partition(), - request->account(), request->value()); + uid, request->name(), request->partition(), request->account(), + request->value()); break; case crane::grpc::ModifyField::DefaultAccount: modify_res = g_account_manager->ModifyUserDefaultAccount( @@ -460,9 +468,10 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( grpc::Status CraneCtldServiceImpl::ModifyQos( grpc::ServerContext *context, const crane::grpc::ModifyQosRequest *request, crane::grpc::ModifyQosReply *response) { - auto modify_res = - g_account_manager->ModifyQos(request->uid(), request->name(), - request->modify_field(), request->value()); + uint32_t uid = ExtractUIDFromMetadata(context); + + auto modify_res = g_account_manager->ModifyQos( + uid, request->name(), request->modify_field(), request->value()); if (modify_res) { response->set_ok(true); @@ -478,9 +487,11 @@ grpc::Status CraneCtldServiceImpl::QueryAccountInfo( grpc::ServerContext *context, const crane::grpc::QueryAccountInfoRequest *request, crane::grpc::QueryAccountInfoReply *response) { + uint32_t uid = ExtractUIDFromMetadata(context); + std::unordered_map res_account_map; - auto modify_res = g_account_manager->QueryAccountInfo( - request->uid(), request->name(), &res_account_map); + auto modify_res = g_account_manager->QueryAccountInfo(uid, request->name(), + &res_account_map); if (modify_res) { response->set_ok(true); } else { @@ -531,9 +542,11 @@ grpc::Status CraneCtldServiceImpl::QueryUserInfo( grpc::ServerContext *context, const crane::grpc::QueryUserInfoRequest *request, crane::grpc::QueryUserInfoReply *response) { + uint32_t uid = ExtractUIDFromMetadata(context); + std::unordered_map res_user_map; - auto modify_res = g_account_manager->QueryUserInfo( - request->uid(), request->name(), &res_user_map); + auto modify_res = + g_account_manager->QueryUserInfo(uid, request->name(), &res_user_map); if (modify_res) { response->set_ok(true); } else { @@ -588,8 +601,10 @@ grpc::Status CraneCtldServiceImpl::QueryQosInfo( crane::grpc::QueryQosInfoReply *response) { std::unordered_map res_qos_map; - auto modify_res = g_account_manager->QueryQosInfo( - request->uid(), request->name(), &res_qos_map); + uint32_t uid = ExtractUIDFromMetadata(context); + + auto modify_res = + g_account_manager->QueryQosInfo(uid, request->name(), &res_qos_map); if (modify_res) { response->set_ok(true); } else { @@ -616,7 +631,9 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( grpc::ServerContext *context, const crane::grpc::DeleteAccountRequest *request, crane::grpc::DeleteAccountReply *response) { - auto res = g_account_manager->DeleteAccount(request->uid(), request->name()); + uint32_t uid = ExtractUIDFromMetadata(context); + + auto res = g_account_manager->DeleteAccount(uid, request->name()); if (res) { response->set_ok(true); } else { @@ -629,8 +646,9 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( grpc::Status CraneCtldServiceImpl::DeleteUser( grpc::ServerContext *context, const crane::grpc::DeleteUserRequest *request, crane::grpc::DeleteUserReply *response) { - auto res = g_account_manager->DeleteUser(request->uid(), request->name(), - request->account()); + uint32_t uid = ExtractUIDFromMetadata(context); + auto res = + g_account_manager->DeleteUser(uid, request->name(), request->account()); if (res) { response->set_ok(true); } else { @@ -644,7 +662,9 @@ grpc::Status CraneCtldServiceImpl::DeleteUser( grpc::Status CraneCtldServiceImpl::DeleteQos( grpc::ServerContext *context, const crane::grpc::DeleteQosRequest *request, crane::grpc::DeleteQosReply *response) { - auto res = g_account_manager->DeleteQos(request->uid(), request->name()); + uint32_t uid = ExtractUIDFromMetadata(context); + + auto res = g_account_manager->DeleteQos(uid, request->name()); if (res) { response->set_ok(true); } else { @@ -659,16 +679,17 @@ grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( grpc::ServerContext *context, const crane::grpc::BlockAccountOrUserRequest *request, crane::grpc::BlockAccountOrUserReply *response) { + uint32_t uid = ExtractUIDFromMetadata(context); AccountManager::CraneExpected res; switch (request->entity_type()) { case crane::grpc::Account: - res = g_account_manager->BlockAccount(request->uid(), request->name(), - request->block()); + res = + g_account_manager->BlockAccount(uid, request->name(), request->block()); break; case crane::grpc::User: - res = g_account_manager->BlockUser(request->uid(), request->name(), - request->account(), request->block()); + res = g_account_manager->BlockUser(uid, request->name(), request->account(), + request->block()); break; default: std::unreachable(); diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index 327500a85..006fa7de0 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -54,8 +54,6 @@ void JwtAuthInterceptor::Intercept( info_->server_context()->TryCancel(); return; } - info_->server_context()->AddInitialMetadata("uid", - util::GetClaim("UID", token)); } } methods->Proceed(); @@ -69,6 +67,14 @@ static std::string GrpcFormatIpAddress(std::string const& addr) { return addr; } +uint32_t ExtractUIDFromMetadata(const grpc::ServerContext* context) { + auto iter = context->client_metadata().find("authorization"); + uint32_t uid = static_cast(std::stoul(util::GetClaim( + "UID", std::string(iter->second.data(), iter->second.size())))); + + return uid; +} + void ServerBuilderSetCompression(grpc::ServerBuilder* builder) { builder->SetDefaultCompressionAlgorithm(GRPC_COMPRESS_GZIP); } diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index 277d1f1ee..01bd74deb 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -78,6 +78,8 @@ class JwtAuthInterceptorFactory std::string jwt_secret_; }; +uint32_t ExtractUIDFromMetadata(const grpc::ServerContext* context); + void ServerBuilderSetCompression(grpc::ServerBuilder* builder); void ServerBuilderSetKeepAliveArgs(grpc::ServerBuilder* builder); From 41aec597da934c42999410b5c6f75a15cb5829de Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:43:21 +0800 Subject: [PATCH 23/62] refactor --- src/CraneCtld/CraneCtld.cpp | 3 +-- src/CraneCtld/CtldPublicDefs.h | 2 -- src/CraneCtld/RpcService/CMakeLists.txt | 2 -- src/CraneCtld/RpcService/CranedKeeper.cpp | 2 -- src/Craned/Craned.cpp | 3 +-- src/Craned/CranedServer.cpp | 3 +-- 6 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index a93a23aa8..cdbfababc 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -24,13 +24,12 @@ #include #include +#include #include "AccountManager.h" #include "AccountMetaContainer.h" #include "CranedKeeper.h" #include "CranedMetaContainer.h" -#include "CtldForCforedServer.h" -#include "CtldForCranedServer.h" #include "CtldGrpcServer.h" #include "CtldPublicDefs.h" #include "DbClient.h" diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 1db77dc81..ad8511647 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -21,8 +21,6 @@ #include "CtldPreCompiledHeader.h" // Precompiled header come first! -#include "crane/GrpcHelper.h" - namespace Ctld { using moodycamel::ConcurrentQueue; diff --git a/src/CraneCtld/RpcService/CMakeLists.txt b/src/CraneCtld/RpcService/CMakeLists.txt index 57f30e599..c0616db85 100644 --- a/src/CraneCtld/RpcService/CMakeLists.txt +++ b/src/CraneCtld/RpcService/CMakeLists.txt @@ -21,8 +21,6 @@ target_link_libraries(RpcService PUBLIC Utility_PublicHeader Utility_PluginClient - dev_event_core - dev_event_pthreads uvw cxxopts diff --git a/src/CraneCtld/RpcService/CranedKeeper.cpp b/src/CraneCtld/RpcService/CranedKeeper.cpp index 85f408546..13941dd7e 100644 --- a/src/CraneCtld/RpcService/CranedKeeper.cpp +++ b/src/CraneCtld/RpcService/CranedKeeper.cpp @@ -18,8 +18,6 @@ #include "CranedKeeper.h" -#include - namespace Ctld { using grpc::ClientContext; diff --git a/src/Craned/Craned.cpp b/src/Craned/Craned.cpp index a165f3a27..9378c2497 100644 --- a/src/Craned/Craned.cpp +++ b/src/Craned/Craned.cpp @@ -29,8 +29,7 @@ #include "CforedClient.h" #include "CranedServer.h" #include "CtldClient.h" -#include "crane/Network.h" -#include "crane/OS.h" +#include "DeviceManager.h" #include "crane/PluginClient.h" #include "crane/String.h" diff --git a/src/Craned/CranedServer.cpp b/src/Craned/CranedServer.cpp index a9785defb..9c26eaa6a 100644 --- a/src/Craned/CranedServer.cpp +++ b/src/Craned/CranedServer.cpp @@ -20,8 +20,7 @@ #include -#include "CranedPublicDefs.h" -#include "CtldClient.h" +#include "TaskManager.h" namespace Craned { From 3f75aa0cba7b5e4cf24fafb877d58c0cd37f284c Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:29:53 +0800 Subject: [PATCH 24/62] refactor --- src/Utilities/PublicHeader/GrpcHelper.cpp | 6 +----- src/Utilities/PublicHeader/include/crane/GrpcHelper.h | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index 006fa7de0..2d058b352 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -132,7 +132,7 @@ void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, const TlsCertificates& certs, - const std::string pem_root_cert) { + const std::string& pem_root_cert) { std::string listen_addr_port = fmt::format("{}:{}", GrpcFormatIpAddress(address), port); @@ -195,10 +195,6 @@ std::shared_ptr CreateTcpInsecureCustomChannel( static void SetSslCredOpts(grpc::SslCredentialsOptions* opts, const TlsCertificates& certs, const ClientTlsCertificates& clientcerts) { - // pem_root_certs is actually the certificate of server side rather than - // CA certificate. CA certificate is not needed. - // Since we use the same cert/key pair for both cranectld/craned, - // pem_root_certs is set to the same certificate. opts->pem_root_certs = clientcerts.ClientCertContent; opts->pem_cert_chain = certs.ServerCertContent; opts->pem_private_key = certs.ServerKeyContent; diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index 01bd74deb..f394753da 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -95,7 +95,7 @@ void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, const TlsCertificates& certs, - const std::string pem_root_cert); + const std::string& pem_root_cert); void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, From e516e7a023c2f678de486886c8770748535fdcc8 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 25 Dec 2024 16:08:34 +0800 Subject: [PATCH 25/62] feat: Add Vault dependencies and Vault encapsulation. --- dependencies/cmake/CMakeLists.txt | 1 + dependencies/cmake/libvault/CMakeLists.txt | 36 ++++++++++ src/CraneCtld/CMakeLists.txt | 2 +- src/CraneCtld/CraneCtld.cpp | 5 ++ src/Utilities/CMakeLists.txt | 3 +- src/Utilities/VaultClient/CMakeLists.txt | 10 +++ src/Utilities/VaultClient/VaultClient.cpp | 66 +++++++++++++++++++ .../VaultClient/include/crane/VaultClient.h | 56 ++++++++++++++++ 8 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 dependencies/cmake/libvault/CMakeLists.txt create mode 100644 src/Utilities/VaultClient/CMakeLists.txt create mode 100644 src/Utilities/VaultClient/VaultClient.cpp create mode 100644 src/Utilities/VaultClient/include/crane/VaultClient.h diff --git a/dependencies/cmake/CMakeLists.txt b/dependencies/cmake/CMakeLists.txt index 188f76f95..49b1d2b35 100644 --- a/dependencies/cmake/CMakeLists.txt +++ b/dependencies/cmake/CMakeLists.txt @@ -28,6 +28,7 @@ add_subdirectory(ranges-v3) add_subdirectory(backward-cpp) add_subdirectory(fpm) add_subdirectory(parallel-hashmap) +add_subdirectory(libvault) include(${CMAKE_SOURCE_DIR}/CMakeModule/SuppressHeaderWarning.cmake) diff --git a/dependencies/cmake/libvault/CMakeLists.txt b/dependencies/cmake/libvault/CMakeLists.txt new file mode 100644 index 000000000..d35db621e --- /dev/null +++ b/dependencies/cmake/libvault/CMakeLists.txt @@ -0,0 +1,36 @@ +include(FetchContent) + +set(LIBVAULT_SRC_URL "https://github.com/abedra/libvault/archive/refs/tags/0.61.0.tar.gz") +# set(CURL_SRC_URL "https://github.com/curl/curl/releases/download/curl-8_11_1/curl-8.11.1.tar.xz") + + +# FetchContent_Declare(curl +# URL ${CURL_SRC_URL} +# URL_HASH SHA256=c7ca7db48b0909743eaef34250da02c19bc61d4f1dcedd6603f109409536ab56 +# INACTIVITY_TIMEOUT 5 +# ) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(libcurl REQUIRED IMPORTED_TARGET libcurl) + +if (libcurl_FOUND) + message(STATUS "Found libcurl ${libcurl_VERSION} in system using pkg-config.") +else () + message(FATAL_ERROR "libcurl in system is not found using pkg-config.") +endif() + +add_library(libcurl INTERFACE) +target_link_libraries(libcurl INTERFACE PkgConfig::libcurl) +target_include_directories(libcurl INTERFACE ${libcurl_INCLUDE_DIRS}) + +set(LIBCURL_BUILD_PRODUCTS ${libcurl_LIBRARY_DIRS}/libcurl.so CACHE STRING "Path to libcurl library" FORCE) +message(STATUS "Using libcurl from system at ${LIBCURL_BUILD_PRODUCTS}.") + +FetchContent_Declare(libvault + URL ${LIBVAULT_SRC_URL} + URL_HASH SHA256=9718ec91157daeb59a47b77447328900d4b9020d0a370d06a2ec012c7017ff78 + INACTIVITY_TIMEOUT 5 + ) + +set(ENABLE_TEST OFF) +FetchContent_MakeAvailable(libvault) \ No newline at end of file diff --git a/src/CraneCtld/CMakeLists.txt b/src/CraneCtld/CMakeLists.txt index 26560d53c..f36b7477d 100644 --- a/src/CraneCtld/CMakeLists.txt +++ b/src/CraneCtld/CMakeLists.txt @@ -30,7 +30,7 @@ target_link_libraries(cranectld PRIVATE RpcService Utility_PublicHeader Utility_PluginClient - + Utility_VaultClient uvw cxxopts diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index cdbfababc..34427fb72 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -37,6 +37,7 @@ #include "TaskScheduler.h" #include "crane/Network.h" #include "crane/PluginClient.h" +#include "crane/VaultClient.h" void ParseConfig(int argc, char** argv) { cxxopts::Options options("cranectld"); @@ -841,6 +842,10 @@ void InitializeCtldGlobalVariables() { g_plugin_client->InitChannelAndStub(g_config.Plugin.PlugindSockPath); } + g_vault_client = std::make_unique( + "REMOVED", "127.0.0.1", "8200"); + g_vault_client->InitPki("crane.com"); + // Account manager must be initialized before Task Scheduler // since the recovery stage of the task scheduler will acquire // information from account manager. diff --git a/src/Utilities/CMakeLists.txt b/src/Utilities/CMakeLists.txt index a561e8fe0..0aee475c3 100644 --- a/src/Utilities/CMakeLists.txt +++ b/src/Utilities/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(AnonymousPipe) add_subdirectory(FileLogger) add_subdirectory(PluginClient) -add_subdirectory(PublicHeader) \ No newline at end of file +add_subdirectory(PublicHeader) +add_subdirectory(VaultClient) \ No newline at end of file diff --git a/src/Utilities/VaultClient/CMakeLists.txt b/src/Utilities/VaultClient/CMakeLists.txt new file mode 100644 index 000000000..c003bf891 --- /dev/null +++ b/src/Utilities/VaultClient/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(Utility_VaultClient + VaultClient.cpp + include/crane/VaultClient.h) +target_include_directories(Utility_VaultClient PUBLIC include) +target_link_libraries(Utility_VaultClient PUBLIC + curl + vault + nlohmann_json + Utility_PublicHeader +) diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp new file mode 100644 index 000000000..254432171 --- /dev/null +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -0,0 +1,66 @@ +/** + * Copyright (c) 2024 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "crane/VaultClient.h" + +namespace vault { + +VaultClient::VaultClient(const std::string& root_token, + const std::string& address, + const std::string& port) + : address_(address), port_(port) { + Vault::TokenStrategy tokenStrategy{Vault::Token{root_token}}; + Vault::Config config = Vault::ConfigBuilder() + .withDebug(false) + .withTlsEnabled(false) + .withHost(Vault::Host{address_}) + .withPort(Vault::Port{port_}) + .build(); + Vault::HttpErrorCallback httpErrorCallback = [&](std::string err) { + std::cout << err << std::endl; + }; + Vault::ResponseErrorCallback responseCallback = [&](Vault::HttpResponse err) { + std::cout << err.statusCode << " : " << err.url.value() << " : " + << err.body.value() << std::endl; + }; + root_client_ = std::make_unique(Vault::Client{ + config, tokenStrategy, httpErrorCallback, responseCallback}); +} + +bool VaultClient::InitPki(const std::string& domains) { + if (!root_client_->is_authenticated()) return false; + + pki_admin_ = std::make_unique(Vault::Pki{*root_client_}); + + return true; +} + +std::expected VaultClient::Sign( + const std::string& csr_content, const std::string& common_name) { + Vault::Parameters parameters( + {{"csr", csr_content}, {"common_name", common_name}}); + + auto response = pki_admin_->sign(Vault::Path{"user"}, parameters); + if (!response) return std::unexpected(false); + + auto data = nlohmann::json::parse(response.value())["data"]; + + return SignResponse{data["serial_number"], data["certificate"], data["issuing_ca"]}; +} + +} // namespace vault \ No newline at end of file diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h new file mode 100644 index 000000000..b9e9887bd --- /dev/null +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2024 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace vault { + +struct SignResponse { + std::string serial_number; + std::string certificate; + std::string issuing_ca; +}; + +class VaultClient { + public: + VaultClient(const std::string& root_token, const std::string& address, + const std::string& port); + bool InitPki(const std::string& domains); + + std::expected Sign(const std::string& csr_content, + const std::string& common_name); + + private: + std::unique_ptr root_client_; + std::unique_ptr pki_admin_; + + std::string address_; + std::string port_; +}; + +} // namespace vault + +inline std::unique_ptr g_vault_client; From cad24faa96ee2dc5d8874564f37b52fa2061255c Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 26 Dec 2024 15:17:29 +0800 Subject: [PATCH 26/62] feat: vault config --- etc/config.yaml | 29 ++++++++++++++++------------- src/CraneCtld/CraneCtld.cpp | 24 +++++++++++++++++++++--- src/CraneCtld/CtldPublicDefs.h | 9 +++++++++ 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/etc/config.yaml b/etc/config.yaml index 41dd1f849..3c528406c 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -13,21 +13,24 @@ CraneBaseDir: /var/crane/ # Tls settings UseTls: true SSL: - CranectldExternalCertFilePath: /etc/crane/cranectld_external.pem - CranectldExternalKeyFilePath: /etc/crane/cranectld_external.key - ExternalCaFilePath: /etc/crane/external_ca.pem - - CranectldInternalCertFilePath: /etc/crane/cranectld_internal.pem - CranectldInternalKeyFilePath: /etc/crane/cranectld_internal.key - CranedCertFilePath: /etc/crane/craned.pem - CranedKeyFilePath: /etc/crane/craned.key - CforedCertFilePath: /etc/crane/cfored.pem - CforedKeyFilePath: /etc/crane/cfored.key - InternalCaFilePath: /etc/crane/internal_ca.pem + CranectldExternalCertFilePath: /etc/crane/tls/external.pem + CranectldExternalKeyFilePath: /etc/crane/tls/external.key + ExternalCaFilePath: /etc/crane/tls/external_ca.pem + CranectldInternalCertFilePath: /etc/crane/tls/cranectld.pem + CranectldInternalKeyFilePath: /etc/crane/tls/cranectld.key + CranedCertFilePath: /etc/crane/tls/craned.pem + CranedKeyFilePath: /etc/crane/tls/craned.key + CforedCertFilePath: /etc/crane/tls/cfored.pem + CforedKeyFilePath: /etc/crane/tls/cfored.key + InternalCaFilePath: /etc/crane/tls/internal_ca.pem + DomainSuffix: crane.com + +Vault: + Addr: 127.0.0.1 + Port: 8200 + Token: REMOVED DomainSuffix: crane.com - -JwtCertFilePath: /etc/crane/jwt.pem # Ctld settings # the listening address of control machine diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 34427fb72..998ff6f41 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -360,6 +360,23 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.UseTls = false; } + if (config["Vault"]) { + const auto& vault_config = config["Vault"]; + + if (vault_config["Addr"]) + g_config.VaultConf.Addr = vault_config["Addr"].as(); + + if (vault_config["Port"]) + g_config.VaultConf.Port = vault_config["Port"].as(); + + if (vault_config["Token"]) + g_config.VaultConf.Token = vault_config["Token"].as(); + + if (vault_config["DomainSuffix"]) + g_config.VaultConf.DomainSuffix = + vault_config["DomainSuffix"].as(); + } + if (config["CraneCtldForeground"]) { g_config.CraneCtldForeground = config["CraneCtldForeground"].as(); } @@ -842,9 +859,10 @@ void InitializeCtldGlobalVariables() { g_plugin_client->InitChannelAndStub(g_config.Plugin.PlugindSockPath); } - g_vault_client = std::make_unique( - "REMOVED", "127.0.0.1", "8200"); - g_vault_client->InitPki("crane.com"); + g_vault_client = std::make_unique( + g_config.VaultConf.Token, g_config.VaultConf.Addr, + g_config.VaultConf.Port); + g_vault_client->InitPki(g_config.VaultConf.DomainSuffix); // Account manager must be initialized before Task Scheduler // since the recovery stage of the task scheduler will acquire diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index ad8511647..89ae3e4ad 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -109,6 +109,15 @@ struct Config { }; CranedListenConf CranedListenConf; + struct VaultConfig { + std::string Addr; + std::string Port; + std::string Token; + std::string DomainSuffix; + }; + + VaultConfig VaultConf; + struct Priority { enum TypeEnum { Basic, MultiFactor }; TypeEnum Type; From b1924d013da4ed1e16b9595ea2532f9b7efd273a Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 26 Dec 2024 16:25:23 +0800 Subject: [PATCH 27/62] feat: SignUserCertificate grpc interface --- protos/Crane.proto | 15 +++++++++++++++ .../RpcService/CtldForCforedServer.cpp | 17 +++++++++++++++++ src/CraneCtld/RpcService/CtldForCforedServer.h | 5 +++++ 3 files changed, 37 insertions(+) diff --git a/protos/Crane.proto b/protos/Crane.proto index f9c832453..4ff315d5c 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -756,6 +756,18 @@ message LoginReply { ErrCode reason = 3; } +message SignUserCertificateRequest { + uint32 uid = 1; + string csr_content = 2; + string alt_names = 3; // "localhost, *.domainSuffix" +} + +message SignUserCertificateResponse { + bool ok = 1; + string certificate = 2; + ErrCode reason = 3; +} + // Todo: Divide service into two parts: one for Craned and one for Crun // We need to distinguish the message sender // and have some kind of authentication @@ -811,6 +823,9 @@ service CraneCtldForCraned { service CraneCtldForCfored { /* RPCs called from Cfored */ rpc CforedStream(stream StreamCforedRequest) returns(stream StreamCtldReply); + + // RPCS called from PKI request + rpc SignUserCertificate(SignUserCertificateRequest) returns (SignUserCertificateResponse); } diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index b52155e1e..4426b2b92 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -207,6 +207,23 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( } } +grpc::Status CtldForCforedServiceImpl::SignUserCertificate( + grpc::ServerContext *context, + const crane::grpc::SignUserCertificateRequest *request, + crane::grpc::SignUserCertificateResponse *response) { + auto result = g_account_manager->SignUserCertificate( + request->uid(), request->csr_content(), request->alt_names()); + if (!result) { + response->set_ok(false); + response->set_reason(result.error()); + } else { + response->set_ok(true); + response->set_certificate(result.value()); + } + + return grpc::Status::OK; +} + void CtldForCforedServer::Shutdown() { m_server_->Shutdown(std::chrono::system_clock::now() + std::chrono::seconds(1)); diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.h b/src/CraneCtld/RpcService/CtldForCforedServer.h index ab9e6bb39..9a302d15f 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.h +++ b/src/CraneCtld/RpcService/CtldForCforedServer.h @@ -180,6 +180,11 @@ class CtldForCforedServiceImpl final crane::grpc::StreamCforedRequest> *stream) override; + grpc::Status SignUserCertificate( + grpc::ServerContext *context, + const crane::grpc::SignUserCertificateRequest *request, + crane::grpc::SignUserCertificateResponse *response) override; + private: CtldForCforedServer *m_ctld_server_; }; From 47e67915d7b99a05cc1bbd978e0a53e4fd78ebbb Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 30 Dec 2024 14:09:15 +0800 Subject: [PATCH 28/62] feat: vault client auto init external --- etc/config.yaml | 10 +- protos/Crane.proto | 3 +- protos/PublicDefs.proto | 2 + src/CraneCtld/AccountManager.cpp | 26 +++ src/CraneCtld/AccountManager.h | 5 + src/CraneCtld/CraneCtld.cpp | 117 ++++++++------ src/CraneCtld/CtldPublicDefs.h | 11 +- src/CraneCtld/RpcService/CMakeLists.txt | 1 + .../RpcService/CtldForCforedServer.cpp | 2 + src/CraneCtld/RpcService/CtldGrpcServer.cpp | 35 ++-- src/CraneCtld/RpcService/CtldGrpcServer.h | 2 +- src/Craned/Craned.cpp | 6 +- src/Craned/CranedPublicDefs.h | 8 +- src/Utilities/PublicHeader/GrpcHelper.cpp | 19 ++- src/Utilities/PublicHeader/OS.cpp | 19 +++ .../PublicHeader/include/crane/GrpcHelper.h | 25 +-- src/Utilities/PublicHeader/include/crane/OS.h | 3 + src/Utilities/VaultClient/VaultClient.cpp | 152 ++++++++++++++++-- .../VaultClient/include/crane/VaultClient.h | 27 +++- 19 files changed, 346 insertions(+), 127 deletions(-) diff --git a/etc/config.yaml b/etc/config.yaml index 3c528406c..73c5736f4 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -13,10 +13,6 @@ CraneBaseDir: /var/crane/ # Tls settings UseTls: true SSL: - CranectldExternalCertFilePath: /etc/crane/tls/external.pem - CranectldExternalKeyFilePath: /etc/crane/tls/external.key - ExternalCaFilePath: /etc/crane/tls/external_ca.pem - CranectldInternalCertFilePath: /etc/crane/tls/cranectld.pem CranectldInternalKeyFilePath: /etc/crane/tls/cranectld.key CranedCertFilePath: /etc/crane/tls/craned.pem @@ -29,9 +25,13 @@ SSL: Vault: Addr: 127.0.0.1 Port: 8200 - Token: REMOVED + Token: REMOVED # Token cannot be leaked DomainSuffix: crane.com + ExternalCertFilePath: /etc/crane/tls/external.pem + ExternalKeyFilePath: /etc/crane/tls/external.key + ExternalCaFilePath: /etc/crane/tls/external_ca.pem + # Ctld settings # the listening address of control machine CraneCtldListenAddr: 0.0.0.0 diff --git a/protos/Crane.proto b/protos/Crane.proto index 4ff315d5c..7b5735643 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -765,7 +765,8 @@ message SignUserCertificateRequest { message SignUserCertificateResponse { bool ok = 1; string certificate = 2; - ErrCode reason = 3; + string external_certificate = 3; + ErrCode reason = 4; } // Todo: Divide service into two parts: one for Craned and one for Crun diff --git a/protos/PublicDefs.proto b/protos/PublicDefs.proto index a06a003c2..46df62b26 100644 --- a/protos/PublicDefs.proto +++ b/protos/PublicDefs.proto @@ -393,6 +393,8 @@ enum ErrCode { ERR_PROTOBUF = 10116; ERR_LIB_EVENT = 10117; ERR_NO_AVAIL_NODE = 10118; + + ERR_SIGN_CERTIFICATE = 10119; } enum EntityType { diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 7738d0d8d..de2e396aa 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1048,6 +1048,32 @@ AccountManager::CraneExpected AccountManager::CheckIfUidHasPermOnUser( return CheckIfUserHasPermOnUserNoLock_(*op_user, user, read_only_priv); } +AccountManager::CraneExpected AccountManager::SignUserCertificate( + uint32_t uid, const std::string& csr_content, + const std::string& alt_names) { + util::write_lock_guard user_guard(m_rw_user_mutex_); + + auto user_result = GetUserInfoByUidNoLock_(uid); + if (!user_result) return std::unexpected(user_result.error()); + const User* op_user = user_result.value(); + + // TODO：验证用户是否存在，以及用户数据库中是否已经有serial_number + std::string common_name = std::format("{}.{}.{}", uid, op_user->name, + g_config.VaultConf.DomainSuffix); + auto sign_response = + g_vault_client->Sign(csr_content, common_name, alt_names); + if (!sign_response) + return std::unexpected(CraneErrCode::ERR_SIGN_CERTIFICATE); + + // TODO:将serial_number保存在数据库中 + + return sign_response->certificate; +} + +/****************************************** + * NOLOCK + ******************************************/ + AccountManager::CraneExpected AccountManager::CheckAddUserAllowedPartitionNoLock_( const User* user, const Account* account, const std::string& partition) { diff --git a/src/CraneCtld/AccountManager.h b/src/CraneCtld/AccountManager.h index a42f259fd..068287e50 100644 --- a/src/CraneCtld/AccountManager.h +++ b/src/CraneCtld/AccountManager.h @@ -23,6 +23,7 @@ #include "DbClient.h" #include "crane/Lock.h" #include "crane/Pointer.h" +#include "crane/VaultClient.h" namespace Ctld { @@ -146,6 +147,10 @@ class AccountManager { const std::string& username, bool read_only_priv); + CraneExpected SignUserCertificate(uint32_t uid, + const std::string& csr_content, + const std::string& alt_names); + private: void InitDataMap_(); diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 998ff6f41..7528159ae 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -186,13 +186,11 @@ void ParseConfig(int argc, char** argv) { if (config["UseTls"] && config["UseTls"].as()) { const auto& ssl_config = config["SSL"]; - TlsCertificates& external_certs = - g_config.ListenConf.TlsCerts.ExternalCerts; - TlsCertificates& internal_certs = + ServerCertificateConfig& internal_certs = g_config.ListenConf.TlsCerts.InternalCerts; - ClientTlsCertificates& craned_certs = + ClientCertificateConfig& craned_certs = g_config.ListenConf.TlsCerts.CranedClientCerts; - ClientTlsCertificates& cfored_certs = + ClientCertificateConfig& cfored_certs = g_config.ListenConf.TlsCerts.CforedClientCerts; g_config.ListenConf.UseTls = true; @@ -201,52 +199,6 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.TlsCerts.DomainSuffix = ssl_config["DomainSuffix"].as(); - if (ssl_config["CranectldExternalCertFilePath"]) { - external_certs.ServerCertFilePath = - ssl_config["CranectldExternalCertFilePath"].as(); - - try { - external_certs.ServerCertContent = - util::ReadFileIntoString(external_certs.ServerCertFilePath); - } catch (const std::exception& e) { - CRANE_ERROR("Read cert file error: {}", e.what()); - std::exit(1); - } - if (external_certs.ServerCertContent.empty()) { - CRANE_ERROR( - "UseTls is true, but the file specified by " - "CranectldExternalCertFilePath " - "is empty"); - } - } else { - CRANE_ERROR( - "UseTls is true, but CranectldExternalCertFilePath is empty"); - std::exit(1); - } - - if (ssl_config["CranectldExternalKeyFilePath"]) { - external_certs.ServerKeyFilePath = - ssl_config["CranectldExternalKeyFilePath"].as(); - - try { - external_certs.ServerKeyContent = - util::ReadFileIntoString(external_certs.ServerKeyFilePath); - } catch (const std::exception& e) { - CRANE_ERROR("Read cert file error: {}", e.what()); - std::exit(1); - } - if (external_certs.ServerKeyContent.empty()) { - CRANE_ERROR( - "UseTls is true, but the file specified by " - "CranectldExternalKeyFilePath " - "is empty"); - } - } else { - CRANE_ERROR( - "UseTls is true, but CranectldExternalKeyFilePath is empty"); - std::exit(1); - } - if (ssl_config["InternalCaFilePath"]) { std::string internalCaFilePath = ssl_config["InternalCaFilePath"].as(); @@ -363,6 +315,11 @@ void ParseConfig(int argc, char** argv) { if (config["Vault"]) { const auto& vault_config = config["Vault"]; + ServerCertificateConfig& external_certs = + g_config.VaultConf.ExternalCerts; + CACertificateConfig& external_ca_certs = + g_config.VaultConf.ExternalCACerts; + if (vault_config["Addr"]) g_config.VaultConf.Addr = vault_config["Addr"].as(); @@ -375,6 +332,54 @@ void ParseConfig(int argc, char** argv) { if (vault_config["DomainSuffix"]) g_config.VaultConf.DomainSuffix = vault_config["DomainSuffix"].as(); + + if (vault_config["ExternalCertFilePath"]) { + external_certs.ServerCertFilePath = + vault_config["ExternalCertFilePath"].as(); + + try { + external_certs.ServerCertContent = + util::ReadFileIntoString(external_certs.ServerCertFilePath); + } catch (const std::exception& e) { + CRANE_DEBUG("Read cert file error: {}", e.what()); + // std::exit(1); + } + } else { + CRANE_ERROR("ExternalCertFilePath is empty"); + std::exit(1); + } + + if (vault_config["ExternalKeyFilePath"]) { + external_certs.ServerKeyFilePath = + vault_config["ExternalKeyFilePath"].as(); + + try { + external_certs.ServerKeyContent = + util::ReadFileIntoString(external_certs.ServerKeyFilePath); + } catch (const std::exception& e) { + CRANE_DEBUG("Read cert file error: {}", e.what()); + // std::exit(1); + } + } else { + CRANE_ERROR("ExternalKeyFilePath is empty"); + std::exit(1); + } + + if (vault_config["ExternalCAFilePath"]) { + external_ca_certs.CACertFilePath = + vault_config["ExternalCAFilePath"].as(); + + try { + external_ca_certs.CACertContent = + util::ReadFileIntoString(external_ca_certs.CACertFilePath); + } catch (const std::exception& e) { + CRANE_DEBUG("Read cert file error: {}", e.what()); + // std::exit(1); + } + } else { + CRANE_ERROR("ExternalCAFilePath is empty"); + std::exit(1); + } } if (config["CraneCtldForeground"]) { @@ -862,7 +867,13 @@ void InitializeCtldGlobalVariables() { g_vault_client = std::make_unique( g_config.VaultConf.Token, g_config.VaultConf.Addr, g_config.VaultConf.Port); - g_vault_client->InitPki(g_config.VaultConf.DomainSuffix); + bool is_init = g_vault_client->InitPki(g_config.VaultConf.DomainSuffix, + &g_config.VaultConf.ExternalCACerts, + &g_config.VaultConf.ExternalCerts); + if (!is_init) { + CRANE_ERROR("vault Init Pki failed"); + std::exit(1); + } // Account manager must be initialized before Task Scheduler // since the recovery stage of the task scheduler will acquire @@ -957,7 +968,7 @@ void InitializeCtldGlobalVariables() { std::exit(1); } - g_ctld_server = std::make_unique(g_config.ListenConf); + g_ctld_server = std::make_unique(); g_ctld_for_craned_server = std::make_unique(g_config.ListenConf); diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 89ae3e4ad..ae141d1e6 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -93,10 +93,10 @@ struct Config { struct TlsCertsConfig { std::string InternalCaContent; std::string DomainSuffix; - TlsCertificates ExternalCerts; - TlsCertificates InternalCerts; - ClientTlsCertificates CranedClientCerts; - ClientTlsCertificates CforedClientCerts; + ServerCertificateConfig ExternalCerts; + ServerCertificateConfig InternalCerts; + ClientCertificateConfig CranedClientCerts; + ClientCertificateConfig CforedClientCerts; }; TlsCertsConfig TlsCerts; @@ -114,6 +114,9 @@ struct Config { std::string Port; std::string Token; std::string DomainSuffix; + + ServerCertificateConfig ExternalCerts; + CACertificateConfig ExternalCACerts; }; VaultConfig VaultConf; diff --git a/src/CraneCtld/RpcService/CMakeLists.txt b/src/CraneCtld/RpcService/CMakeLists.txt index c0616db85..0ea0444a1 100644 --- a/src/CraneCtld/RpcService/CMakeLists.txt +++ b/src/CraneCtld/RpcService/CMakeLists.txt @@ -20,6 +20,7 @@ target_link_libraries(RpcService PUBLIC Utility_PublicHeader Utility_PluginClient + Utility_VaultClient uvw diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index 4426b2b92..52313071a 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -219,6 +219,8 @@ grpc::Status CtldForCforedServiceImpl::SignUserCertificate( } else { response->set_ok(true); response->set_certificate(result.value()); + response->set_external_certificate( + g_config.VaultConf.ExternalCerts.ServerCertContent); } return grpc::Status::OK; diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index 96341b161..a0cbd33c1 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -713,31 +713,28 @@ grpc::Status CraneCtldServiceImpl::QueryClusterInfo( return grpc::Status::OK; } -CtldServer::CtldServer(const Config::CraneCtldListenConf &listen_conf) { +CtldServer::CtldServer() : m_service_impl_(nullptr) { m_service_impl_ = std::make_unique(this); grpc::ServerBuilder builder; - + const auto &listen_conf = g_config.ListenConf; + const auto &vault_conf = g_config.VaultConf; if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); std::string cranectld_listen_addr = listen_conf.CraneCtldListenAddr; - if (listen_conf.UseTls) { - ServerBuilderAddTcpTlsListeningPort(&builder, cranectld_listen_addr, - listen_conf.CraneCtldListenPort, - listen_conf.TlsCerts.ExternalCerts); - } else { - ServerBuilderAddTcpInsecureListeningPort(&builder, cranectld_listen_addr, - listen_conf.CraneCtldListenPort); - } - - std::vector< - std::unique_ptr> - creators; - creators.push_back( - std::unique_ptr( - new JwtAuthInterceptorFactory(g_config.ListenConf.JwtSecretContent))); - - builder.experimental().SetInterceptorCreators(std::move(creators)); + ServerBuilderAddmTcpTlsListeningPort( + &builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort, + vault_conf.ExternalCerts, vault_conf.ExternalCACerts.CACertContent); + + // std::vector< + // std::unique_ptr> + // creators; + // creators.push_back( + // std::unique_ptr( + // new + // JwtAuthInterceptorFactory(g_config.ListenConf.JwtSecretContent))); + + // builder.experimental().SetInterceptorCreators(std::move(creators)); builder.RegisterService(m_service_impl_.get()); diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.h b/src/CraneCtld/RpcService/CtldGrpcServer.h index b5c38242c..71ae71c26 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.h +++ b/src/CraneCtld/RpcService/CtldGrpcServer.h @@ -157,7 +157,7 @@ class CtldServer { * User must make sure that this constructor is called only once! * @param listen_address The "[Address]:[Port]" of CraneCtld. */ - explicit CtldServer(const Config::CraneCtldListenConf &listen_conf); + explicit CtldServer(); inline void Wait() { m_server_->Wait(); } diff --git a/src/Craned/Craned.cpp b/src/Craned/Craned.cpp index 9378c2497..201835ede 100644 --- a/src/Craned/Craned.cpp +++ b/src/Craned/Craned.cpp @@ -173,11 +173,11 @@ void ParseConfig(int argc, char** argv) { const auto& ssl_config = config["SSL"]; g_config.ListenConf.UseTls = true; - TlsCertificates& craned_certs = + ServerCertificateConfig& craned_certs = g_config.ListenConf.TlsCerts.CranedTlsCerts; - ClientTlsCertificates& internal_client_certs = + ClientCertificateConfig& internal_client_certs = g_config.ListenConf.TlsCerts.InternalClientTlsCerts; - ClientTlsCertificates& cfoed_client_certs = + ClientCertificateConfig& cfoed_client_certs = g_config.ListenConf.TlsCerts.CforedClientTlsCerts; if (ssl_config["DomainSuffix"]) diff --git a/src/Craned/CranedPublicDefs.h b/src/Craned/CranedPublicDefs.h index 323fadaac..b15a870b6 100644 --- a/src/Craned/CranedPublicDefs.h +++ b/src/Craned/CranedPublicDefs.h @@ -58,11 +58,11 @@ struct Config { struct TlsCertsConfig { std::string InternalCaContent; std::string DomainSuffix; - TlsCertificates CranedTlsCerts; - ClientTlsCertificates InternalClientTlsCerts; - ClientTlsCertificates CforedClientTlsCerts; + ServerCertificateConfig CranedTlsCerts; + ClientCertificateConfig InternalClientTlsCerts; + ClientCertificateConfig CforedClientTlsCerts; }; - + TlsCertsConfig TlsCerts; std::string UnixSocketListenAddr; diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index 2d058b352..c679fde1d 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -111,7 +111,7 @@ void ServerBuilderAddTcpInsecureListeningPort(grpc::ServerBuilder* builder, void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, - const TlsCertificates& certs) { + const ServerCertificateConfig& certs) { std::string listen_addr_port = fmt::format("{}:{}", GrpcFormatIpAddress(address), port); @@ -131,7 +131,7 @@ void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, - const TlsCertificates& certs, + const ServerCertificateConfig& certs, const std::string& pem_root_cert) { std::string listen_addr_port = fmt::format("{}:{}", GrpcFormatIpAddress(address), port); @@ -193,8 +193,8 @@ std::shared_ptr CreateTcpInsecureCustomChannel( } static void SetSslCredOpts(grpc::SslCredentialsOptions* opts, - const TlsCertificates& certs, - const ClientTlsCertificates& clientcerts) { + const ServerCertificateConfig& certs, + const ClientCertificateConfig& clientcerts) { opts->pem_root_certs = clientcerts.ClientCertContent; opts->pem_cert_chain = certs.ServerCertContent; opts->pem_private_key = certs.ServerKeyContent; @@ -202,7 +202,8 @@ static void SetSslCredOpts(grpc::SslCredentialsOptions* opts, std::shared_ptr CreateTcpTlsCustomChannelByIp( const std::string& ip, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const ServerCertificateConfig& certs, + const ClientCertificateConfig& clientcerts, const grpc::ChannelArguments& args) { grpc::SslCredentialsOptions ssl_opts; SetSslCredOpts(&ssl_opts, certs, clientcerts); @@ -214,7 +215,8 @@ std::shared_ptr CreateTcpTlsCustomChannelByIp( std::shared_ptr CreateTcpTlsChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const ServerCertificateConfig& certs, + const ClientCertificateConfig& clientcerts, const std::string& domainSuffix) { grpc::SslCredentialsOptions ssl_opts; SetSslCredOpts(&ssl_opts, certs, clientcerts); @@ -225,8 +227,9 @@ std::shared_ptr CreateTcpTlsChannelByHostname( std::shared_ptr CreateTcpTlsCustomChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, - const std::string& domainSuffix, const grpc::ChannelArguments& args) { + const ServerCertificateConfig& certs, + const ClientCertificateConfig& clientcerts, const std::string& domainSuffix, + const grpc::ChannelArguments& args) { grpc::SslCredentialsOptions ssl_opts; SetSslCredOpts(&ssl_opts, certs, clientcerts); diff --git a/src/Utilities/PublicHeader/OS.cpp b/src/Utilities/PublicHeader/OS.cpp index 127ac0fbc..c6f127a43 100644 --- a/src/Utilities/PublicHeader/OS.cpp +++ b/src/Utilities/PublicHeader/OS.cpp @@ -38,6 +38,25 @@ bool DeleteFile(std::string const& p) { return ok; } +bool SaveFile(std::string const& p, std::string const& content) { + try { + if (std::filesystem::exists(p)) return true; + + std::ofstream file(p); + if (!file.is_open()) { + CRANE_ERROR("Failed to open file {}", p); + return false; + } + file << content; + file.close(); + } catch (const std::exception& e) { + CRANE_ERROR("Failed to save file {}: {}", p, e.what()); + return false; + } + + return true; +} + bool CreateFolders(std::string const& p) { if (std::filesystem::exists(p)) return true; diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index f394753da..8170fc490 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -27,19 +27,23 @@ #include "crane/Jwt.h" #include "crane/Network.h" -struct TlsCertificates { +struct ServerCertificateConfig { std::string ServerCertFilePath; std::string ServerCertContent; std::string ServerKeyFilePath; std::string ServerKeyContent; - std::string DomainSuffix; }; -struct ClientTlsCertificates { +struct ClientCertificateConfig { std::string ClientCertFilePath; std::string ClientCertContent; }; +struct CACertificateConfig { + std::string CACertFilePath; + std::string CACertContent; +}; + // class JwtAuthProcessor : public grpc::AuthMetadataProcessor { // public: // JwtAuthProcessor(std::string secret) : jwt_secret_(secret) {} @@ -94,13 +98,13 @@ void ServerBuilderAddTcpInsecureListeningPort(grpc::ServerBuilder* builder, void ServerBuilderAddmTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, - const TlsCertificates& certs, + const ServerCertificateConfig& certs, const std::string& pem_root_cert); void ServerBuilderAddTcpTlsListeningPort(grpc::ServerBuilder* builder, const std::string& address, const std::string& port, - const TlsCertificates& certs); + const ServerCertificateConfig& certs); void SetGrpcClientKeepAliveChannelArgs(grpc::ChannelArguments* args); @@ -120,15 +124,18 @@ std::shared_ptr CreateTcpInsecureCustomChannel( std::shared_ptr CreateTcpTlsCustomChannelByIp( const std::string& ip, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const ServerCertificateConfig& certs, + const ClientCertificateConfig& clientcerts, const grpc::ChannelArguments& args); std::shared_ptr CreateTcpTlsChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, + const ServerCertificateConfig& certs, + const ClientCertificateConfig& clientcerts, const std::string& domainSuffix); std::shared_ptr CreateTcpTlsCustomChannelByHostname( const std::string& hostname, const std::string& port, - const TlsCertificates& certs, const ClientTlsCertificates& clientcerts, - const std::string& domainSuffix, const grpc::ChannelArguments& args); + const ServerCertificateConfig& certs, + const ClientCertificateConfig& clientcerts, const std::string& domainSuffix, + const grpc::ChannelArguments& args); diff --git a/src/Utilities/PublicHeader/include/crane/OS.h b/src/Utilities/PublicHeader/include/crane/OS.h index 03f05fa23..cd95f38ac 100644 --- a/src/Utilities/PublicHeader/include/crane/OS.h +++ b/src/Utilities/PublicHeader/include/crane/OS.h @@ -24,6 +24,7 @@ #include #include +#include #include "crane/Logger.h" #include "crane/OS.h" @@ -40,6 +41,8 @@ namespace os { bool DeleteFile(std::string const& p); +bool SaveFile(std::string const& p, std::string const& content); + bool CreateFolders(std::string const& p); bool CreateFoldersForFile(std::string const& p); diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index 254432171..648f05a81 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -21,8 +21,7 @@ namespace vault { VaultClient::VaultClient(const std::string& root_token, - const std::string& address, - const std::string& port) + const std::string& address, const std::string& port) : address_(address), port_(port) { Vault::TokenStrategy tokenStrategy{Vault::Token{root_token}}; Vault::Config config = Vault::ConfigBuilder() @@ -32,35 +31,160 @@ VaultClient::VaultClient(const std::string& root_token, .withPort(Vault::Port{port_}) .build(); Vault::HttpErrorCallback httpErrorCallback = [&](std::string err) { - std::cout << err << std::endl; + CRANE_DEBUG(err); }; Vault::ResponseErrorCallback responseCallback = [&](Vault::HttpResponse err) { - std::cout << err.statusCode << " : " << err.url.value() << " : " - << err.body.value() << std::endl; + CRANE_DEBUG("{} : {}", err.url.value(), err.body.value()); }; root_client_ = std::make_unique(Vault::Client{ config, tokenStrategy, httpErrorCallback, responseCallback}); } -bool VaultClient::InitPki(const std::string& domains) { +bool VaultClient::InitPki(const std::string& domains, + CACertificateConfig* external_ca, + ServerCertificateConfig* external_cert) { if (!root_client_->is_authenticated()) return false; - pki_admin_ = std::make_unique(Vault::Pki{*root_client_}); + pki_root_ = std::make_unique( + Vault::Pki{*root_client_, Vault::SecretMount{"pki"}}); + pki_internal_ = std::make_unique( + Vault::Pki{*root_client_, Vault::SecretMount{"pki_internal"}}); + pki_external_ = std::make_unique( + Vault::Pki{*root_client_, Vault::SecretMount{"pki_external"}}); + + // 检查CA是否存在，存在则写入文件中，不存在则创建CA + if (!IssureExternalCa_(domains, external_ca)) return false; + + // 创建role + if (!CreateRole_("external", domains)) return false; + + // 检查external_pem，external_key是否存在，不存在则创建，写入文件中 + if (!IssureExternalCert_("external", domains, external_cert)) return false; return true; } std::expected VaultClient::Sign( - const std::string& csr_content, const std::string& common_name) { - Vault::Parameters parameters( - {{"csr", csr_content}, {"common_name", common_name}}); + const std::string& csr_content, const std::string& common_name, + const std::string& alt_names) { + Vault::Parameters parameters({{"csr", csr_content}, + {"common_name", common_name}, + {"alt_names", alt_names}, + {"exclude_cn_from_sans", "true"}}); + + nlohmann::json::value_type data; + try { + auto response = pki_external_->sign(Vault::Path{"external"}, parameters); + if (!response) return std::unexpected(false); + + data = nlohmann::json::parse(response.value())["data"]; + + } catch (const std::exception& e) { + return std::unexpected(false); + } + + return SignResponse{data["serial_number"], data["certificate"]}; +} + +bool VaultClient::IssureExternalCa_(const std::string& domains, + CACertificateConfig* external_ca) { + nlohmann::json::value_type data; + std::optional response; + + if (!external_ca->CACertContent.empty()) return true; + + try { + response = pki_external_->readCACertificate(); + if (!response) return false; + + std::string external_ca_pem = response.value(); + + if (external_ca_pem == "") { + response = pki_external_->generateIntermediate( + Vault::KeyType{"internal"}, + Vault::Parameters({{"common_name", std::format("*.{}", domains)}, + {"issuer_name", "CraneSched_internal_CA"}, + {"not_after", "2030-12-31T23:59:59Z"}})); + + data = nlohmann::json::parse(response.value())["data"]; + std::string external_ca_csr = data["csr"]; + + response = pki_root_->signIntermediate( + Vault::Parameters({{"csr", external_ca_csr}, + {"format", "pem_bundle"}, + {"not_after", "2030-12-31T23:59:59Z"}})); + if (!response) return false; + + data = nlohmann::json::parse(response.value())["data"]; + + external_ca_pem = data["certificate"]; - auto response = pki_admin_->sign(Vault::Path{"user"}, parameters); - if (!response) return std::unexpected(false); + response = pki_external_->setSignedIntermediate( + Vault::Parameters({{"certificate", external_ca_pem}})); + if (!response) return false; + } - auto data = nlohmann::json::parse(response.value())["data"]; + if (!util::os::SaveFile(external_ca->CACertFilePath, external_ca_pem)) + return false; - return SignResponse{data["serial_number"], data["certificate"], data["issuing_ca"]}; + } catch (const std::exception& e) { + return false; + } + + return true; +} + +bool VaultClient::CreateRole_(const std::string& role_name, + const std::string& domains) { + nlohmann::json::value_type data; + std::optional response; + + try { + response = pki_external_->createRole( + Vault::Path{role_name}, + Vault::Parameters{{"allowed_domains", domains}, + {"allow_subdomains", "true"}, + {"allow_glob_domains", "true"}, + {"not_after", "2030-12-31T23:59:59Z"}}); + if (!response) return false; + } catch (const std::exception& e) { + return false; + } + + return true; +} + +bool VaultClient::IssureExternalCert_(const std::string& role_name, + const std::string& domains, + ServerCertificateConfig* external_cert) { + nlohmann::json::value_type data; + std::optional response; + + if (!external_cert->ServerCertContent.empty()) return true; + + try { + response = pki_external_->issue( + Vault::Path(role_name), + Vault::Parameters{{"common_name", std::format("external.{}", domains)}, + {"alt_names", std::format("localhost,*.{}", domains)}, + {"exclude_cn_from_sans", "true"}}); + if (!response) return false; + + data = nlohmann::json::parse(response.value())["data"]; + std::string external_pem = data["certificate"]; + std::string external_key = data["private_key"]; + + if (!util::os::SaveFile(external_cert->ServerCertFilePath, external_pem)) + return false; + + if (!util::os::SaveFile(external_cert->ServerKeyFilePath, external_key)) + return false; + + } catch (const std::exception& e) { + return false; + } + + return true; } } // namespace vault \ No newline at end of file diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index b9e9887bd..f315d0769 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -21,31 +21,46 @@ #include #include -#include #include #include #include +#include "crane/GrpcHelper.h" +#include "crane/Logger.h" +#include "crane/OS.h" + namespace vault { struct SignResponse { std::string serial_number; std::string certificate; - std::string issuing_ca; }; class VaultClient { public: VaultClient(const std::string& root_token, const std::string& address, - const std::string& port); - bool InitPki(const std::string& domains); + const std::string& port); + bool InitPki(const std::string& domains, CACertificateConfig* external_ca, + ServerCertificateConfig* external_cert); std::expected Sign(const std::string& csr_content, - const std::string& common_name); + const std::string& common_name, + const std::string& alt_names); private: + bool IssureExternalCa_(const std::string& domains, + CACertificateConfig* external_ca); + + bool CreateRole_(const std::string& role_name, const std::string& domains); + + bool IssureExternalCert_(const std::string& role_name, + const std::string& domains, + ServerCertificateConfig* external_cert); + std::unique_ptr root_client_; - std::unique_ptr pki_admin_; + std::unique_ptr pki_root_; + std::unique_ptr pki_internal_; + std::unique_ptr pki_external_; std::string address_; std::string port_; From b9e1676748314016f22b4f15360e148173fccd3b Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 30 Dec 2024 14:58:53 +0800 Subject: [PATCH 29/62] fix: cert content empty --- src/CraneCtld/CraneCtld.cpp | 9 ++-- src/Utilities/PublicHeader/OS.cpp | 2 + src/Utilities/VaultClient/VaultClient.cpp | 52 +++++++++++++++++++---- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 7528159ae..96c8d9267 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -867,13 +867,10 @@ void InitializeCtldGlobalVariables() { g_vault_client = std::make_unique( g_config.VaultConf.Token, g_config.VaultConf.Addr, g_config.VaultConf.Port); - bool is_init = g_vault_client->InitPki(g_config.VaultConf.DomainSuffix, - &g_config.VaultConf.ExternalCACerts, - &g_config.VaultConf.ExternalCerts); - if (!is_init) { - CRANE_ERROR("vault Init Pki failed"); + if (!g_vault_client->InitPki(g_config.VaultConf.DomainSuffix, + &g_config.VaultConf.ExternalCACerts, + &g_config.VaultConf.ExternalCerts)) std::exit(1); - } // Account manager must be initialized before Task Scheduler // since the recovery stage of the task scheduler will acquire diff --git a/src/Utilities/PublicHeader/OS.cpp b/src/Utilities/PublicHeader/OS.cpp index c6f127a43..b2860f7ae 100644 --- a/src/Utilities/PublicHeader/OS.cpp +++ b/src/Utilities/PublicHeader/OS.cpp @@ -39,6 +39,8 @@ bool DeleteFile(std::string const& p) { } bool SaveFile(std::string const& p, std::string const& content) { + if (!CreateFoldersForFile(p)) return false; + try { if (std::filesystem::exists(p)) return true; diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index 648f05a81..3fd7ace2e 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -43,7 +43,10 @@ VaultClient::VaultClient(const std::string& root_token, bool VaultClient::InitPki(const std::string& domains, CACertificateConfig* external_ca, ServerCertificateConfig* external_cert) { - if (!root_client_->is_authenticated()) return false; + if (!root_client_->is_authenticated()) { + CRANE_ERROR("Root client is not authenticated"); + return false; + } pki_root_ = std::make_unique( Vault::Pki{*root_client_, Vault::SecretMount{"pki"}}); @@ -80,6 +83,7 @@ std::expected VaultClient::Sign( data = nlohmann::json::parse(response.value())["data"]; } catch (const std::exception& e) { + CRANE_DEBUG("Failed to sign certificate: {}", e.what()); return std::unexpected(false); } @@ -95,7 +99,10 @@ bool VaultClient::IssureExternalCa_(const std::string& domains, try { response = pki_external_->readCACertificate(); - if (!response) return false; + if (!response) { + CRANE_ERROR("Failed to read external CA certificate"); + return false; + } std::string external_ca_pem = response.value(); @@ -104,7 +111,7 @@ bool VaultClient::IssureExternalCa_(const std::string& domains, Vault::KeyType{"internal"}, Vault::Parameters({{"common_name", std::format("*.{}", domains)}, {"issuer_name", "CraneSched_internal_CA"}, - {"not_after", "2030-12-31T23:59:59Z"}})); + {"not_after", "9999-12-31T23:59:59Z"}})); data = nlohmann::json::parse(response.value())["data"]; std::string external_ca_csr = data["csr"]; @@ -112,8 +119,11 @@ bool VaultClient::IssureExternalCa_(const std::string& domains, response = pki_root_->signIntermediate( Vault::Parameters({{"csr", external_ca_csr}, {"format", "pem_bundle"}, - {"not_after", "2030-12-31T23:59:59Z"}})); - if (!response) return false; + {"not_after", "9999-12-31T23:59:59Z"}})); + if (!response) { + CRANE_ERROR("Failed to sign external CA certificate"); + return false; + } data = nlohmann::json::parse(response.value())["data"]; @@ -121,13 +131,23 @@ bool VaultClient::IssureExternalCa_(const std::string& domains, response = pki_external_->setSignedIntermediate( Vault::Parameters({{"certificate", external_ca_pem}})); - if (!response) return false; + if (!response) { + CRANE_ERROR("Failed to set signed external CA certificate"); + return false; + } + } + + external_ca->CACertContent = external_ca_pem; + if (external_ca->CACertFilePath.empty()) { + CRANE_ERROR("ExternalCAFilePath is empty"); + return false; } if (!util::os::SaveFile(external_ca->CACertFilePath, external_ca_pem)) return false; } catch (const std::exception& e) { + CRANE_ERROR("Failed to issue external CA certificate: {}", e.what()); return false; } @@ -140,14 +160,21 @@ bool VaultClient::CreateRole_(const std::string& role_name, std::optional response; try { + response = pki_external_->readRole(Vault::Path{role_name}); + if (response) return true; + response = pki_external_->createRole( Vault::Path{role_name}, Vault::Parameters{{"allowed_domains", domains}, {"allow_subdomains", "true"}, {"allow_glob_domains", "true"}, - {"not_after", "2030-12-31T23:59:59Z"}}); - if (!response) return false; + {"not_after", "9999-12-31T23:59:59Z"}}); + if (!response) { + CRANE_ERROR("Failed to create role {}", role_name); + return false; + } } catch (const std::exception& e) { + CRANE_ERROR("Failed to create role {}: {}", role_name, e.what()); return false; } @@ -168,12 +195,18 @@ bool VaultClient::IssureExternalCert_(const std::string& role_name, Vault::Parameters{{"common_name", std::format("external.{}", domains)}, {"alt_names", std::format("localhost,*.{}", domains)}, {"exclude_cn_from_sans", "true"}}); - if (!response) return false; + if (!response) { + CRANE_ERROR("Failed to issue external certificate"); + return false; + } data = nlohmann::json::parse(response.value())["data"]; std::string external_pem = data["certificate"]; std::string external_key = data["private_key"]; + external_cert->ServerCertContent = external_pem; + external_cert->ServerKeyContent = external_key; + if (!util::os::SaveFile(external_cert->ServerCertFilePath, external_pem)) return false; @@ -181,6 +214,7 @@ bool VaultClient::IssureExternalCert_(const std::string& role_name, return false; } catch (const std::exception& e) { + CRANE_ERROR("Failed to issue external certificate: {}", e.what()); return false; } From b44f5d6735ca1ce6b7fb3e15faa7731deadb8066 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 30 Dec 2024 15:12:31 +0800 Subject: [PATCH 30/62] feat: add permissions with save file --- src/Utilities/PublicHeader/OS.cpp | 10 +++++++++- src/Utilities/PublicHeader/include/crane/OS.h | 4 +++- src/Utilities/VaultClient/VaultClient.cpp | 8 +++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/Utilities/PublicHeader/OS.cpp b/src/Utilities/PublicHeader/OS.cpp index b2860f7ae..c0788688e 100644 --- a/src/Utilities/PublicHeader/OS.cpp +++ b/src/Utilities/PublicHeader/OS.cpp @@ -38,7 +38,8 @@ bool DeleteFile(std::string const& p) { return ok; } -bool SaveFile(std::string const& p, std::string const& content) { +bool SaveFile(std::string const& p, std::string const& content, + mode_t permissions = 0644) { if (!CreateFoldersForFile(p)) return false; try { @@ -51,6 +52,13 @@ bool SaveFile(std::string const& p, std::string const& content) { } file << content; file.close(); + + if (chmod(p.c_str(), permissions) != 0) { + CRANE_ERROR("Failed to set permissions for file {}: {}", p, + strerror(errno)); + return false; + } + } catch (const std::exception& e) { CRANE_ERROR("Failed to save file {}: {}", p, e.what()); return false; diff --git a/src/Utilities/PublicHeader/include/crane/OS.h b/src/Utilities/PublicHeader/include/crane/OS.h index cd95f38ac..0c6e3f596 100644 --- a/src/Utilities/PublicHeader/include/crane/OS.h +++ b/src/Utilities/PublicHeader/include/crane/OS.h @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -41,7 +42,8 @@ namespace os { bool DeleteFile(std::string const& p); -bool SaveFile(std::string const& p, std::string const& content); +bool SaveFile(std::string const& p, std::string const& content, + mode_t permissions); bool CreateFolders(std::string const& p); diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index 3fd7ace2e..4577ee1b7 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -143,7 +143,7 @@ bool VaultClient::IssureExternalCa_(const std::string& domains, return false; } - if (!util::os::SaveFile(external_ca->CACertFilePath, external_ca_pem)) + if (!util::os::SaveFile(external_ca->CACertFilePath, external_ca_pem, 0644)) return false; } catch (const std::exception& e) { @@ -207,10 +207,12 @@ bool VaultClient::IssureExternalCert_(const std::string& role_name, external_cert->ServerCertContent = external_pem; external_cert->ServerKeyContent = external_key; - if (!util::os::SaveFile(external_cert->ServerCertFilePath, external_pem)) + if (!util::os::SaveFile(external_cert->ServerCertFilePath, external_pem, + 0644)) return false; - if (!util::os::SaveFile(external_cert->ServerKeyFilePath, external_key)) + if (!util::os::SaveFile(external_cert->ServerKeyFilePath, external_key, + 0600)) return false; } catch (const std::exception& e) { From ce1ac85d9f32b608501e48446907a154bad5dfca Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 30 Dec 2024 15:55:36 +0800 Subject: [PATCH 31/62] refactor --- src/CraneCtld/CraneCtld.cpp | 21 ------------------- .../RpcService/CtldForCforedServer.cpp | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 96c8d9267..bda5977a7 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -133,27 +133,6 @@ void ParseConfig(int argc, char** argv) { g_config.CraneCtldMutexFilePath = g_config.CraneBaseDir + kDefaultCraneCtldMutexFile; - if (config["JwtCertFilePath"]) { - std::string jwtCertFilePath = - config["JwtCertFilePath"].as(); - - try { - g_config.ListenConf.JwtSecretContent = - util::ReadFileIntoString(jwtCertFilePath); - } catch (const std::exception& e) { - CRANE_ERROR("Read cert file error: {}", e.what()); - std::exit(1); - } - if (g_config.ListenConf.JwtSecretContent.empty()) { - CRANE_ERROR( - "The file specified by JwtCertFilePath " - "is empty"); - } - } else { - CRANE_ERROR("JwtCertFilePath is empty"); - std::exit(1); - } - if (config["CraneCtldListenAddr"]) g_config.ListenConf.CraneCtldListenAddr = config["CraneCtldListenAddr"].as(); diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index 52313071a..2c326a732 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -220,7 +220,7 @@ grpc::Status CtldForCforedServiceImpl::SignUserCertificate( response->set_ok(true); response->set_certificate(result.value()); response->set_external_certificate( - g_config.VaultConf.ExternalCerts.ServerCertContent); + g_config.VaultConf.ExternalCACerts.CACertContent); } return grpc::Status::OK; From a192f7243170aeace0fb8fa4a492c0106a7e467c Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 30 Dec 2024 17:00:57 +0800 Subject: [PATCH 32/62] merge master --- src/CraneCtld/RpcService/CMakeLists.txt | 1 - .../RpcService/CtldForCforedServer.cpp | 47 ++++----- .../RpcService/CtldForCforedServer.h | 8 +- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 95 ++++++++++--------- src/CraneCtld/TaskScheduler.cpp | 34 +++---- src/CraneCtld/TaskScheduler.h | 2 +- 6 files changed, 95 insertions(+), 92 deletions(-) diff --git a/src/CraneCtld/RpcService/CMakeLists.txt b/src/CraneCtld/RpcService/CMakeLists.txt index 0ea0444a1..018ec4bb7 100644 --- a/src/CraneCtld/RpcService/CMakeLists.txt +++ b/src/CraneCtld/RpcService/CMakeLists.txt @@ -39,7 +39,6 @@ target_link_libraries(RpcService PUBLIC mongocxx_static range-v3::range-v3 - result Backward::Interface ) diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index 2c326a732..523dbcce9 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -64,21 +64,22 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( CRANE_ERROR("Expect type CFORED_REGISTRATION from peer {}.", context->peer()); return Status::CANCELLED; - } else { - cfored_name = cfored_request.payload_cfored_reg().cfored_name(); - CRANE_INFO("Cfored {} registered.", cfored_name); + } - ok = stream_writer->WriteCforedRegistrationAck({}); - if (ok) { - state = StreamState::kWaitMsg; - } else { - CRANE_ERROR( - "Failed to send msg to cfored {}. Connection is broken. " - "Exiting...", - cfored_name); - state = StreamState::kCleanData; - } + cfored_name = cfored_request.payload_cfored_reg().cfored_name(); + CRANE_INFO("Cfored {} registered.", cfored_name); + + ok = stream_writer->WriteCforedRegistrationAck({}); + if (ok) { + state = StreamState::kWaitMsg; + } else { + CRANE_ERROR( + "Failed to send msg to cfored {}. Connection is broken. " + "Exiting...", + cfored_name); + state = StreamState::kCleanData; } + } else { state = StreamState::kCleanData; } @@ -108,15 +109,16 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( }; meta.cb_task_cancel = [writer_weak_ptr](task_id_t task_id) { + CRANE_TRACE("Sending TaskCancelRequest in task_cancel", task_id); if (auto writer = writer_weak_ptr.lock(); writer) writer->WriteTaskCancelRequest(task_id); }; - meta.cb_task_completed = [this, i_type, cfored_name, - writer_weak_ptr](task_id_t task_id) { - CRANE_TRACE("Sending TaskCompletionAckReply in task_completed", - task_id); - if (auto writer = writer_weak_ptr.lock(); writer) + meta.cb_task_completed = [this, i_type, cfored_name, writer_weak_ptr]( + task_id_t task_id, + bool send_completion_ack) { + if (auto writer = writer_weak_ptr.lock(); + writer && send_completion_ack) writer->WriteTaskCompletionAckReply(task_id); m_ctld_server_->m_mtx_.Lock(); @@ -134,12 +136,12 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( auto submit_result = g_task_scheduler->SubmitTaskToScheduler(std::move(task)); - result::result result; + std::expected result; if (submit_result.has_value()) { - result = result::result{ + result = std::expected{ submit_result.value().get()}; } else { - result = result::fail(submit_result.error()); + result = std::unexpected(submit_result.error()); } ok = stream_writer->WriteTaskIdReply(payload.pid(), result); @@ -162,8 +164,7 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( case StreamCforedRequest::TASK_COMPLETION_REQUEST: { auto const &payload = cfored_request.payload_task_complete_req(); CRANE_TRACE("Recv TaskCompletionReq of Task #{}", payload.task_id()); - - if (g_task_scheduler->TerminatePendingOrRunningTask( + if (g_task_scheduler->TerminatePendingOrRunningIaTask( payload.task_id()) != CraneErr::kOk) stream_writer->WriteTaskCompletionAckReply(payload.task_id()); } break; diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.h b/src/CraneCtld/RpcService/CtldForCforedServer.h index 9a302d15f..e18048e3a 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.h +++ b/src/CraneCtld/RpcService/CtldForCforedServer.h @@ -43,7 +43,7 @@ class CforedStreamWriter { : m_stream_(stream), m_valid_(true) {} bool WriteTaskIdReply(pid_t calloc_pid, - result::result res) { + std::expected res) { LockGuard guard(&m_stream_mtx_); if (!m_valid_) return false; @@ -65,8 +65,7 @@ class CforedStreamWriter { bool WriteTaskResAllocReply( task_id_t task_id, - result::result>, - std::string> + std::expected>, std::string> res) { LockGuard guard(&m_stream_mtx_); if (!m_valid_) return false; @@ -119,8 +118,7 @@ class CforedStreamWriter { return m_stream_->Write(reply); } - bool WriteCforedRegistrationAck( - const result::result &res) { + bool WriteCforedRegistrationAck(const std::expected &res) { LockGuard guard(&m_stream_mtx_); if (!m_valid_) return false; diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index a0cbd33c1..e96f5cbf4 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -119,9 +119,9 @@ grpc::Status CraneCtldServiceImpl::ModifyTask( grpc::ServerContext *context, const crane::grpc::ModifyTaskRequest *request, crane::grpc::ModifyTaskReply *response) { using ModifyTaskRequest = crane::grpc::ModifyTaskRequest; - uint32_t uid = ExtractUIDFromMetadata(context); - auto res = g_account_manager->CheckUidIsAdmin(uid); - if (res.has_error()) { + // uint32_t uid = ExtractUIDFromMetadata(context); + auto res = g_account_manager->CheckUidIsAdmin(request->uid()); + if (!res) { for (auto task_id : request->task_ids()) { response->add_not_modified_tasks(task_id); response->add_not_modified_reasons(res.error()); @@ -279,7 +279,7 @@ grpc::Status CraneCtldServiceImpl::Login( grpc::Status CraneCtldServiceImpl::AddAccount( grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); Account account; const crane::grpc::AccountInfo *account_info = &request->account(); @@ -295,7 +295,7 @@ grpc::Status CraneCtldServiceImpl::AddAccount( account.allowed_qos_list.emplace_back(qos); } - auto result = g_account_manager->AddAccount(uid, account); + auto result = g_account_manager->AddAccount(request->uid(), account); if (result) { response->set_ok(true); } else { @@ -309,7 +309,7 @@ grpc::Status CraneCtldServiceImpl::AddAccount( grpc::Status CraneCtldServiceImpl::AddUser( grpc::ServerContext *context, const crane::grpc::AddUserRequest *request, crane::grpc::AddUserReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); User user; const crane::grpc::UserInfo *user_info = &request->user(); @@ -339,7 +339,7 @@ grpc::Status CraneCtldServiceImpl::AddUser( } AccountManager::CraneExpected result = - g_account_manager->AddUser(uid, user); + g_account_manager->AddUser(request->uid(), user); if (result) { response->set_ok(true); } else { @@ -353,7 +353,7 @@ grpc::Status CraneCtldServiceImpl::AddUser( grpc::Status CraneCtldServiceImpl::AddQos( grpc::ServerContext *context, const crane::grpc::AddQosRequest *request, crane::grpc::AddQosReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); Qos qos; const crane::grpc::QosInfo *qos_info = &request->qos(); @@ -373,7 +373,7 @@ grpc::Status CraneCtldServiceImpl::AddQos( } qos.max_time_limit_per_task = absl::Seconds(sec); - auto result = g_account_manager->AddQos(uid, qos); + auto result = g_account_manager->AddQos(request->uid(), qos); if (result) { response->set_ok(true); } else { @@ -388,10 +388,10 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( grpc::ServerContext *context, const crane::grpc::ModifyAccountRequest *request, crane::grpc::ModifyAccountReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); auto modify_res = g_account_manager->ModifyAccount( - request->type(), uid, request->name(), request->modify_field(), + request->type(), request->uid(), request->name(), request->modify_field(), request->value(), request->force()); if (modify_res) { @@ -407,7 +407,7 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( grpc::Status CraneCtldServiceImpl::ModifyUser( grpc::ServerContext *context, const crane::grpc::ModifyUserRequest *request, crane::grpc::ModifyUserReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); AccountManager::CraneExpected modify_res; @@ -415,12 +415,13 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( switch (request->modify_field()) { case crane::grpc::ModifyField::Partition: modify_res = g_account_manager->DeleteUserAllowedPartition( - uid, request->name(), request->account(), request->value()); + request->uid(), request->name(), request->account(), + request->value()); break; case crane::grpc::ModifyField::Qos: modify_res = g_account_manager->DeleteUserAllowedQos( - uid, request->name(), request->partition(), request->account(), - request->value(), request->force()); + request->uid(), request->name(), request->partition(), + request->account(), request->value(), request->force()); break; default: std::unreachable(); @@ -428,23 +429,24 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( } else { switch (request->modify_field()) { case crane::grpc::ModifyField::AdminLevel: - modify_res = g_account_manager->ModifyAdminLevel(uid, request->name(), - request->value()); + modify_res = g_account_manager->ModifyAdminLevel( + request->uid(), request->name(), request->value()); break; case crane::grpc::ModifyField::Partition: modify_res = g_account_manager->ModifyUserAllowedPartition( - request->type(), uid, request->name(), request->account(), + request->type(), request->uid(), request->name(), request->account(), request->value()); break; case crane::grpc::ModifyField::Qos: modify_res = g_account_manager->ModifyUserAllowedQos( - request->type(), uid, request->name(), request->partition(), - request->account(), request->value(), request->force()); + request->type(), request->uid(), request->name(), + request->partition(), request->account(), request->value(), + request->force()); break; case crane::grpc::ModifyField::DefaultQos: modify_res = g_account_manager->ModifyUserDefaultQos( - uid, request->name(), request->partition(), request->account(), - request->value()); + request->uid(), request->name(), request->partition(), + request->account(), request->value()); break; case crane::grpc::ModifyField::DefaultAccount: modify_res = g_account_manager->ModifyUserDefaultAccount( @@ -468,10 +470,11 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( grpc::Status CraneCtldServiceImpl::ModifyQos( grpc::ServerContext *context, const crane::grpc::ModifyQosRequest *request, crane::grpc::ModifyQosReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); - auto modify_res = g_account_manager->ModifyQos( - uid, request->name(), request->modify_field(), request->value()); + auto modify_res = + g_account_manager->ModifyQos(request->uid(), request->name(), + request->modify_field(), request->value()); if (modify_res) { response->set_ok(true); @@ -487,11 +490,11 @@ grpc::Status CraneCtldServiceImpl::QueryAccountInfo( grpc::ServerContext *context, const crane::grpc::QueryAccountInfoRequest *request, crane::grpc::QueryAccountInfoReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); std::unordered_map res_account_map; - auto modify_res = g_account_manager->QueryAccountInfo(uid, request->name(), - &res_account_map); + auto modify_res = g_account_manager->QueryAccountInfo( + request->uid(), request->name(), &res_account_map); if (modify_res) { response->set_ok(true); } else { @@ -542,11 +545,11 @@ grpc::Status CraneCtldServiceImpl::QueryUserInfo( grpc::ServerContext *context, const crane::grpc::QueryUserInfoRequest *request, crane::grpc::QueryUserInfoReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); std::unordered_map res_user_map; - auto modify_res = - g_account_manager->QueryUserInfo(uid, request->name(), &res_user_map); + auto modify_res = g_account_manager->QueryUserInfo( + request->uid(), request->name(), &res_user_map); if (modify_res) { response->set_ok(true); } else { @@ -601,10 +604,10 @@ grpc::Status CraneCtldServiceImpl::QueryQosInfo( crane::grpc::QueryQosInfoReply *response) { std::unordered_map res_qos_map; - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); - auto modify_res = - g_account_manager->QueryQosInfo(uid, request->name(), &res_qos_map); + auto modify_res = g_account_manager->QueryQosInfo( + request->uid(), request->name(), &res_qos_map); if (modify_res) { response->set_ok(true); } else { @@ -631,9 +634,9 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( grpc::ServerContext *context, const crane::grpc::DeleteAccountRequest *request, crane::grpc::DeleteAccountReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); - auto res = g_account_manager->DeleteAccount(uid, request->name()); + auto res = g_account_manager->DeleteAccount(request->uid(), request->name()); if (res) { response->set_ok(true); } else { @@ -646,9 +649,9 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( grpc::Status CraneCtldServiceImpl::DeleteUser( grpc::ServerContext *context, const crane::grpc::DeleteUserRequest *request, crane::grpc::DeleteUserReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); - auto res = - g_account_manager->DeleteUser(uid, request->name(), request->account()); + // uint32_t uid = ExtractUIDFromMetadata(context); + auto res = g_account_manager->DeleteUser(request->uid(), request->name(), + request->account()); if (res) { response->set_ok(true); } else { @@ -662,9 +665,9 @@ grpc::Status CraneCtldServiceImpl::DeleteUser( grpc::Status CraneCtldServiceImpl::DeleteQos( grpc::ServerContext *context, const crane::grpc::DeleteQosRequest *request, crane::grpc::DeleteQosReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); - auto res = g_account_manager->DeleteQos(uid, request->name()); + auto res = g_account_manager->DeleteQos(request->uid(), request->name()); if (res) { response->set_ok(true); } else { @@ -679,17 +682,17 @@ grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( grpc::ServerContext *context, const crane::grpc::BlockAccountOrUserRequest *request, crane::grpc::BlockAccountOrUserReply *response) { - uint32_t uid = ExtractUIDFromMetadata(context); + // uint32_t uid = ExtractUIDFromMetadata(context); AccountManager::CraneExpected res; switch (request->entity_type()) { case crane::grpc::Account: - res = - g_account_manager->BlockAccount(uid, request->name(), request->block()); + res = g_account_manager->BlockAccount(request->uid(), request->name(), + request->block()); break; case crane::grpc::User: - res = g_account_manager->BlockUser(uid, request->name(), request->account(), - request->block()); + res = g_account_manager->BlockUser(request->uid(), request->name(), + request->account(), request->block()); break; default: std::unreachable(); diff --git a/src/CraneCtld/TaskScheduler.cpp b/src/CraneCtld/TaskScheduler.cpp index c02ad7de7..9d2388d97 100644 --- a/src/CraneCtld/TaskScheduler.cpp +++ b/src/CraneCtld/TaskScheduler.cpp @@ -1041,12 +1041,12 @@ void TaskScheduler::SetNodeSelectionAlgo( m_node_selection_algo_ = std::move(algo); } -result::result, std::string> +std::expected, std::string> TaskScheduler::SubmitTaskToScheduler(std::unique_ptr task) { CraneErr err; if (!task->password_entry->Valid()) { - return result::fail( + return std::unexpected( fmt::format("Uid {} not found on the controller node", task->uid)); } task->SetUsername(task->password_entry->Username()); @@ -1055,7 +1055,7 @@ TaskScheduler::SubmitTaskToScheduler(std::unique_ptr task) { auto user_scoped_ptr = g_account_manager->GetExistedUserInfo(task->Username()); if (!user_scoped_ptr) { - return result::fail(fmt::format( + return std::unexpected(fmt::format( "User '{}' not found in the account database", task->Username())); } @@ -1064,7 +1064,7 @@ TaskScheduler::SubmitTaskToScheduler(std::unique_ptr task) { task->MutableTaskToCtld()->set_account(user_scoped_ptr->default_account); } else { if (!user_scoped_ptr->account_to_attrs_map.contains(task->account)) { - return result::fail(fmt::format( + return std::unexpected(fmt::format( "Account '{}' is not in your account list", task->account)); } } @@ -1072,7 +1072,7 @@ TaskScheduler::SubmitTaskToScheduler(std::unique_ptr task) { if (!g_account_manager->CheckUserPermissionToPartition( task->Username(), task->account, task->partition_id)) { - return result::fail( + return std::unexpected( fmt::format("User '{}' doesn't have permission to use partition '{}' " "when using account '{}'", task->Username(), task->partition_id, task->account)); @@ -1080,48 +1080,50 @@ TaskScheduler::SubmitTaskToScheduler(std::unique_ptr task) { auto enable_res = g_account_manager->CheckIfUserOfAccountIsEnabled( task->Username(), task->account); - if (enable_res.has_error()) { - return result::fail(enable_res.error()); + if (!enable_res) { + return std::unexpected(enable_res.error()); } - err = AcquireTaskAttributes(task.get()); + err = g_task_scheduler->AcquireTaskAttributes(task.get()); - if (err == CraneErr::kOk) err = CheckTaskValidity(task.get()); + if (err == CraneErr::kOk) + err = g_task_scheduler->CheckTaskValidity(task.get()); if (err == CraneErr::kOk) { task->SetSubmitTime(absl::Now()); - std::future future = SubmitTaskAsync(std::move(task)); + std::future future = + g_task_scheduler->SubmitTaskAsync(std::move(task)); return {std::move(future)}; } if (err == CraneErr::kNonExistent) { CRANE_DEBUG("Task submission failed. Reason: Partition doesn't exist!"); - return result::fail("Partition doesn't exist!"); + return std::unexpected("Partition doesn't exist!"); } else if (err == CraneErr::kInvalidNodeNum) { CRANE_DEBUG( "Task submission failed. Reason: --node is either invalid or greater " "than the number of nodes in its partition."); - return result::fail( + return std::unexpected( "--node is either invalid or greater than the number of nodes in its " "partition."); } else if (err == CraneErr::kNoResource) { CRANE_DEBUG( "Task submission failed. " "Reason: The resources of the partition are insufficient."); - return result::fail("The resources of the partition are insufficient"); + return std::unexpected("The resources of the partition are insufficient"); } else if (err == CraneErr::kNoAvailNode) { CRANE_DEBUG( "Task submission failed. " "Reason: Nodes satisfying the requirements of task are insufficient"); - return result::fail( + return std::unexpected( "Nodes satisfying the requirements of task are insufficient."); } else if (err == CraneErr::kInvalidParam) { CRANE_DEBUG( "Task submission failed. " "Reason: The param of task is invalid."); - return result::fail("The param of task is invalid."); + return std::unexpected("The param of task is invalid."); } - return result::fail(CraneErrStr(err)); + return std::unexpected(CraneErrStr(err)); } std::future TaskScheduler::SubmitTaskAsync( diff --git a/src/CraneCtld/TaskScheduler.h b/src/CraneCtld/TaskScheduler.h index efe0567a9..ebf80cde9 100644 --- a/src/CraneCtld/TaskScheduler.h +++ b/src/CraneCtld/TaskScheduler.h @@ -445,7 +445,7 @@ class TaskScheduler { void SetNodeSelectionAlgo(std::unique_ptr algo); - result::result, std::string> SubmitTaskToScheduler( + std::expected, std::string> SubmitTaskToScheduler( std::unique_ptr task); /// \return The future is set to 0 if task submission is failed. From 05a756a1dc39e9beb789e067e54991612208fd5d Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 31 Dec 2024 10:17:26 +0800 Subject: [PATCH 33/62] feat: user database add serial number --- protos/Crane.proto | 14 -------- protos/PublicDefs.proto | 1 + src/CraneCtld/AccountManager.cpp | 39 ++++++++++----------- src/CraneCtld/AccountManager.h | 2 -- src/CraneCtld/CtldPublicDefs.h | 2 +- src/CraneCtld/DbClient.cpp | 7 ++-- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 16 --------- src/CraneCtld/RpcService/CtldGrpcServer.h | 4 --- 8 files changed, 24 insertions(+), 61 deletions(-) diff --git a/protos/Crane.proto b/protos/Crane.proto index 7b5735643..cb5ecd799 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -745,17 +745,6 @@ message StreamTaskIOReply { } } -message LoginRequest { - uint32 uid = 1; - string password = 2; -} - -message LoginReply { - bool ok = 1; - string token = 2; - ErrCode reason = 3; -} - message SignUserCertificateRequest { uint32 uid = 1; string csr_content = 2; @@ -808,9 +797,6 @@ service CraneCtld { /* RPCs called from cinfo */ rpc QueryClusterInfo(QueryClusterInfoRequest) returns (QueryClusterInfoReply); - /* RPCs called from clogin */ - rpc Login(LoginRequest) returns (LoginReply); - /* common RPCs */ rpc QueryTasksInfo(QueryTasksInfoRequest) returns (QueryTasksInfoReply); } diff --git a/protos/PublicDefs.proto b/protos/PublicDefs.proto index 46df62b26..8a089c6fc 100644 --- a/protos/PublicDefs.proto +++ b/protos/PublicDefs.proto @@ -395,6 +395,7 @@ enum ErrCode { ERR_NO_AVAIL_NODE = 10118; ERR_SIGN_CERTIFICATE = 10119; + ERR_DUPLICATE_CERTIFICATE = 10120; } enum EntityType { diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index de2e396aa..4f5cd6f71 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -26,25 +26,6 @@ namespace Ctld { AccountManager::AccountManager() { InitDataMap_(); } -AccountManager::CraneExpected AccountManager::Login( - uint32_t uid, const std::string& password) { - util::read_lock_guard user_guard(m_rw_user_mutex_); - - auto user_result = GetUserInfoByUidNoLock_(uid); - if (!user_result) return std::unexpected(user_result.error()); - const User* user = user_result.value(); - - if (password != user->password) { - return std::unexpected(CraneErrCode::ERR_PASSWORD_MISMATCH); - } - std::unordered_map claims{ - {"UID", std::to_string(uid)}}; - const std::string& token = - util::GenerateToken(g_config.ListenConf.JwtSecretContent, claims); - - return token; -} - AccountManager::CraneExpected AccountManager::AddUser( uint32_t uid, const User& new_user) { CraneExpected result; @@ -1057,7 +1038,10 @@ AccountManager::CraneExpected AccountManager::SignUserCertificate( if (!user_result) return std::unexpected(user_result.error()); const User* op_user = user_result.value(); - // TODO：验证用户是否存在，以及用户数据库中是否已经有serial_number + // Verify whether the serial number already exists in the user database. + if (!op_user->serial_number.empty()) + return std::unexpected(CraneErrCode::ERR_DUPLICATE_CERTIFICATE); + std::string common_name = std::format("{}.{}.{}", uid, op_user->name, g_config.VaultConf.DomainSuffix); auto sign_response = @@ -1065,7 +1049,20 @@ AccountManager::CraneExpected AccountManager::SignUserCertificate( if (!sign_response) return std::unexpected(CraneErrCode::ERR_SIGN_CERTIFICATE); - // TODO:将serial_number保存在数据库中 + // Save the serial number in the database. + mongocxx::client_session::with_transaction_cb callback = + [&](mongocxx::client_session* session) { + g_db_client->UpdateEntityOne(MongodbClient::EntityType::USER, "$set", + op_user->name, "serial_number", + sign_response->serial_number); + }; + + // Update to database + if (!g_db_client->CommitTransaction(callback)) { + return std::unexpected(CraneErrCode::ERR_UPDATE_DATABASE); + } + + m_user_map_[op_user->name]->serial_number = sign_response->serial_number; return sign_response->certificate; } diff --git a/src/CraneCtld/AccountManager.h b/src/CraneCtld/AccountManager.h index 068287e50..e58f64a54 100644 --- a/src/CraneCtld/AccountManager.h +++ b/src/CraneCtld/AccountManager.h @@ -50,8 +50,6 @@ class AccountManager { ~AccountManager() = default; - CraneExpected Login(uint32_t uid, const std::string& password); - CraneExpected AddUser(uint32_t uid, const User& new_user); CraneExpected AddAccount(uint32_t uid, const Account& new_account); diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index ae141d1e6..5a30e7689 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -733,8 +733,8 @@ struct User { bool deleted = false; uid_t uid; std::string name; - std::string password; std::string default_account; + std::string serial_number; AccountToAttrsMap account_to_attrs_map; std::list coordinator_accounts; AdminLevel admin_level; diff --git a/src/CraneCtld/DbClient.cpp b/src/CraneCtld/DbClient.cpp index f51553a1f..2ec9db8f9 100644 --- a/src/CraneCtld/DbClient.cpp +++ b/src/CraneCtld/DbClient.cpp @@ -95,7 +95,6 @@ bool MongodbClient::CheckDefaultRootAccountUserAndInit_() { CRANE_TRACE("Default user ROOT not found. Insert it into DB."); root_user.name = "root"; - root_user.password = ""; root_user.default_account = "ROOT"; root_user.admin_level = User::Root; root_user.uid = 0; @@ -687,19 +686,21 @@ void MongodbClient::ViewToUser_(const bsoncxx::document::view& user_view, bsoncxx::builder::basic::document MongodbClient::UserToDocument_( const Ctld::User& user) { - std::array fields{"deleted", + std::array fields{"deleted", "uid", "default_account", "name", + "serial_number", "admin_level", "account_to_attrs_map", "coordinator_accounts"}; - std::tuple> values{false, user.uid, user.default_account, user.name, + user.serial_number, user.admin_level, user.account_to_attrs_map, user.coordinator_accounts}; diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index e96f5cbf4..be810f59b 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -261,21 +261,6 @@ grpc::Status CraneCtldServiceImpl::QueryTasksInfo( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::Login( - grpc::ServerContext *context, const crane::grpc::LoginRequest *request, - crane::grpc::LoginReply *response) { - auto result = g_account_manager->Login(request->uid(), request->password()); - - if (result) { - response->set_ok(true); - response->set_token(result.value()); - } else { - response->set_ok(false); - response->set_reason(result.error()); - } - return grpc::Status::OK; -} - grpc::Status CraneCtldServiceImpl::AddAccount( grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) { @@ -315,7 +300,6 @@ grpc::Status CraneCtldServiceImpl::AddUser( const crane::grpc::UserInfo *user_info = &request->user(); user.name = user_info->name(); - user.password = user_info->password(); user.uid = user_info->uid(); user.default_account = user_info->account(); user.admin_level = User::AdminLevel(user_info->admin_level()); diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.h b/src/CraneCtld/RpcService/CtldGrpcServer.h index 71ae71c26..892194c23 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.h +++ b/src/CraneCtld/RpcService/CtldGrpcServer.h @@ -78,10 +78,6 @@ class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { const crane::grpc::ModifyCranedStateRequest *request, crane::grpc::ModifyCranedStateReply *response) override; - grpc::Status Login(grpc::ServerContext *context, - const crane::grpc::LoginRequest *request, - crane::grpc::LoginReply *response) override; - grpc::Status AddAccount(grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) override; From f94f1757bb27d823e292a9ad391fa6f8e36c574b Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:15:21 +0800 Subject: [PATCH 34/62] feat: revoke cert --- src/Utilities/VaultClient/VaultClient.cpp | 28 +++++++++++++++++++ .../VaultClient/include/crane/VaultClient.h | 8 ++++++ 2 files changed, 36 insertions(+) diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index 4577ee1b7..f23692f9f 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -154,6 +154,34 @@ bool VaultClient::IssureExternalCa_(const std::string& domains, return true; } +bool VaultClient::RevokeCert(const std::string& serial_number) { + try { + auto response = pki_external_->revokeCertificate( + Vault::Parameters{{"serial_number", serial_number}}); + if (!response) return false; + } catch (const std::exception& e) { + return false; + } + + allowed_certs_.erase(serial_number); + + return true; +} + +bool VaultClient::IsCertAllowed(const std::string& serial_number) { + if (allowed_certs_.contains(serial_number)) return true; + + try { + // TODO: 检查crl列表 + } catch (const std::exception& e) { + return false; + } + + allowed_certs_.emplace(serial_number); + + return true; +} + bool VaultClient::CreateRole_(const std::string& role_name, const std::string& domains) { nlohmann::json::value_type data; diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index f315d0769..b418d9ed1 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -24,6 +24,7 @@ #include #include #include +#include #include "crane/GrpcHelper.h" #include "crane/Logger.h" @@ -36,6 +37,8 @@ struct SignResponse { std::string certificate; }; +using AllowedCerts = std::unordered_set; + class VaultClient { public: VaultClient(const std::string& root_token, const std::string& address, @@ -47,6 +50,10 @@ class VaultClient { const std::string& common_name, const std::string& alt_names); + bool RevokeCert(const std::string& serial_number); + + bool IsCertAllowed(const std::string& serial_number); + private: bool IssureExternalCa_(const std::string& domains, CACertificateConfig* external_ca); @@ -62,6 +69,7 @@ class VaultClient { std::unique_ptr pki_internal_; std::unique_ptr pki_external_; + AllowedCerts allowed_certs_; std::string address_; std::string port_; }; From 084c7216bc039a30456d7ac90efa06173f9553a6 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:37:08 +0800 Subject: [PATCH 35/62] feat: extract uid from cert --- src/Utilities/PublicHeader/GrpcHelper.cpp | 20 ++++++++++---- src/Utilities/PublicHeader/String.cpp | 27 +++++++++++++++++++ .../PublicHeader/include/crane/GrpcHelper.h | 6 ++++- .../PublicHeader/include/crane/String.h | 5 ++++ 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index c679fde1d..1fc5fc281 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -67,12 +67,22 @@ static std::string GrpcFormatIpAddress(std::string const& addr) { return addr; } -uint32_t ExtractUIDFromMetadata(const grpc::ServerContext* context) { - auto iter = context->client_metadata().find("authorization"); - uint32_t uid = static_cast(std::stoul(util::GetClaim( - "UID", std::string(iter->second.data(), iter->second.size())))); +// Retrieve UID from the certificate. +std::expected ExtractUIDFromCert( + const grpc::ServerContext* context) { + auto cert = context->auth_context()->FindPropertyValues("x509_pem_cert"); + if (cert.empty()) return std::unexpected(false); - return uid; + std::string certificate = std::string(cert[0].data(), cert[0].size()); + + auto result = util::ParseCertificate(certificate); + if (!result) return std::unexpected(false); + + std::vector cn_parts = absl::StrSplit(result.value(), '.'); + if (cn_parts.size() != 3 || cn_parts[0].empty()) + return std::unexpected(false); + + return static_cast(std::stoul(cn_parts[0])); } void ServerBuilderSetCompression(grpc::ServerBuilder* builder) { diff --git a/src/Utilities/PublicHeader/String.cpp b/src/Utilities/PublicHeader/String.cpp index 6fc988270..91452d509 100644 --- a/src/Utilities/PublicHeader/String.cpp +++ b/src/Utilities/PublicHeader/String.cpp @@ -411,4 +411,31 @@ std::string GenerateCommaSeparatedString(const int val) { return absl::StrJoin(val_vec, ","); } +std::expected ParseCertificate(const std::string &cert_pem) { + // Load the certificate content into a BIO (memory buffer). + BIO *bio = BIO_new_mem_buf(cert_pem.data(), cert_pem.size()); + if (!bio) return std::unexpected(false); + + // Read a PEM-formatted certificate. + X509 *cert = PEM_read_bio_X509(bio, nullptr, nullptr, nullptr); + if (!cert) { + BIO_free(bio); + return std::unexpected(false); + } + + // Retrieve Subject information. + X509_NAME *subject = X509_get_subject_name(cert); + if (!subject) return std::unexpected(false); + + char cn[256]; + int len = X509_NAME_get_text_by_NID(subject, NID_commonName, cn, sizeof(cn)); + if (len <= 0) return std::unexpected(false); + + // free the memory + X509_free(cert); + BIO_free(bio); + + return cn; +} + } // namespace util diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index 8170fc490..cd7cabdc6 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -24,8 +24,11 @@ #include #include +#include + #include "crane/Jwt.h" #include "crane/Network.h" +#include "crane/String.h" struct ServerCertificateConfig { std::string ServerCertFilePath; @@ -82,7 +85,8 @@ class JwtAuthInterceptorFactory std::string jwt_secret_; }; -uint32_t ExtractUIDFromMetadata(const grpc::ServerContext* context); +std::expected ExtractUIDFromCert( + const grpc::ServerContext* context); void ServerBuilderSetCompression(grpc::ServerBuilder* builder); diff --git a/src/Utilities/PublicHeader/include/crane/String.h b/src/Utilities/PublicHeader/include/crane/String.h index a89cd9eea..2cc8f581f 100644 --- a/src/Utilities/PublicHeader/include/crane/String.h +++ b/src/Utilities/PublicHeader/include/crane/String.h @@ -20,10 +20,13 @@ #include #include +#include +#include #include #include #include +#include #include #include #include @@ -82,4 +85,6 @@ std::string ReadableGrpcDresInNode( std::string GenerateCommaSeparatedString(const int val); +std::expected ParseCertificate(const std::string &cert_pem); + } // namespace util \ No newline at end of file From 86f828dbffaff64e67d562fdb480e242d448f3ec Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:42:08 +0800 Subject: [PATCH 36/62] feat: use uid from cert --- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 209 +++++++++++++++----- 1 file changed, 160 insertions(+), 49 deletions(-) diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index be810f59b..fe764508e 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -23,6 +23,7 @@ #include "../EmbeddedDbClient.h" #include "../TaskScheduler.h" #include "CranedKeeper.h" +#include "crane/GrpcHelper.h" #include "crane/String.h" namespace Ctld { @@ -119,8 +120,18 @@ grpc::Status CraneCtldServiceImpl::ModifyTask( grpc::ServerContext *context, const crane::grpc::ModifyTaskRequest *request, crane::grpc::ModifyTaskReply *response) { using ModifyTaskRequest = crane::grpc::ModifyTaskRequest; - // uint32_t uid = ExtractUIDFromMetadata(context); - auto res = g_account_manager->CheckUidIsAdmin(request->uid()); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + for (auto task_id : request->task_ids()) { + response->add_not_modified_tasks(task_id); + response->add_not_modified_reasons("uid is invalid."); + } + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); + + auto res = g_account_manager->CheckUidIsAdmin(uid); if (!res) { for (auto task_id : request->task_ids()) { response->add_not_modified_tasks(task_id); @@ -202,7 +213,18 @@ grpc::Status CraneCtldServiceImpl::ModifyNode( grpc::ServerContext *context, const crane::grpc::ModifyCranedStateRequest *request, crane::grpc::ModifyCranedStateReply *response) { - auto res = g_account_manager->CheckUidIsAdmin(request->uid()); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + for (auto crane_id : request->craned_ids()) { + response->add_not_modified_nodes(crane_id); + response->add_not_modified_reasons("uid is invalid."); + } + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); + + auto res = g_account_manager->CheckUidIsAdmin(uid); if (!res) { for (auto crane_id : request->craned_ids()) { response->add_not_modified_nodes(crane_id); @@ -264,7 +286,14 @@ grpc::Status CraneCtldServiceImpl::QueryTasksInfo( grpc::Status CraneCtldServiceImpl::AddAccount( grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); Account account; const crane::grpc::AccountInfo *account_info = &request->account(); @@ -280,7 +309,7 @@ grpc::Status CraneCtldServiceImpl::AddAccount( account.allowed_qos_list.emplace_back(qos); } - auto result = g_account_manager->AddAccount(request->uid(), account); + auto result = g_account_manager->AddAccount(uid, account); if (result) { response->set_ok(true); } else { @@ -294,7 +323,14 @@ grpc::Status CraneCtldServiceImpl::AddAccount( grpc::Status CraneCtldServiceImpl::AddUser( grpc::ServerContext *context, const crane::grpc::AddUserRequest *request, crane::grpc::AddUserReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); User user; const crane::grpc::UserInfo *user_info = &request->user(); @@ -323,7 +359,7 @@ grpc::Status CraneCtldServiceImpl::AddUser( } AccountManager::CraneExpected result = - g_account_manager->AddUser(request->uid(), user); + g_account_manager->AddUser(uid, user); if (result) { response->set_ok(true); } else { @@ -337,7 +373,14 @@ grpc::Status CraneCtldServiceImpl::AddUser( grpc::Status CraneCtldServiceImpl::AddQos( grpc::ServerContext *context, const crane::grpc::AddQosRequest *request, crane::grpc::AddQosReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); Qos qos; const crane::grpc::QosInfo *qos_info = &request->qos(); @@ -357,7 +400,7 @@ grpc::Status CraneCtldServiceImpl::AddQos( } qos.max_time_limit_per_task = absl::Seconds(sec); - auto result = g_account_manager->AddQos(request->uid(), qos); + auto result = g_account_manager->AddQos(uid, qos); if (result) { response->set_ok(true); } else { @@ -372,10 +415,17 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( grpc::ServerContext *context, const crane::grpc::ModifyAccountRequest *request, crane::grpc::ModifyAccountReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); auto modify_res = g_account_manager->ModifyAccount( - request->type(), request->uid(), request->name(), request->modify_field(), + request->type(), uid, request->name(), request->modify_field(), request->value(), request->force()); if (modify_res) { @@ -391,7 +441,14 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( grpc::Status CraneCtldServiceImpl::ModifyUser( grpc::ServerContext *context, const crane::grpc::ModifyUserRequest *request, crane::grpc::ModifyUserReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); AccountManager::CraneExpected modify_res; @@ -399,13 +456,12 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( switch (request->modify_field()) { case crane::grpc::ModifyField::Partition: modify_res = g_account_manager->DeleteUserAllowedPartition( - request->uid(), request->name(), request->account(), - request->value()); + uid, request->name(), request->account(), request->value()); break; case crane::grpc::ModifyField::Qos: modify_res = g_account_manager->DeleteUserAllowedQos( - request->uid(), request->name(), request->partition(), - request->account(), request->value(), request->force()); + uid, request->name(), request->partition(), request->account(), + request->value(), request->force()); break; default: std::unreachable(); @@ -413,24 +469,23 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( } else { switch (request->modify_field()) { case crane::grpc::ModifyField::AdminLevel: - modify_res = g_account_manager->ModifyAdminLevel( - request->uid(), request->name(), request->value()); + modify_res = g_account_manager->ModifyAdminLevel(uid, request->name(), + request->value()); break; case crane::grpc::ModifyField::Partition: modify_res = g_account_manager->ModifyUserAllowedPartition( - request->type(), request->uid(), request->name(), request->account(), + request->type(), uid, request->name(), request->account(), request->value()); break; case crane::grpc::ModifyField::Qos: modify_res = g_account_manager->ModifyUserAllowedQos( - request->type(), request->uid(), request->name(), - request->partition(), request->account(), request->value(), - request->force()); + request->type(), uid, request->name(), request->partition(), + request->account(), request->value(), request->force()); break; case crane::grpc::ModifyField::DefaultQos: modify_res = g_account_manager->ModifyUserDefaultQos( - request->uid(), request->name(), request->partition(), - request->account(), request->value()); + uid, request->name(), request->partition(), request->account(), + request->value()); break; case crane::grpc::ModifyField::DefaultAccount: modify_res = g_account_manager->ModifyUserDefaultAccount( @@ -454,11 +509,17 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( grpc::Status CraneCtldServiceImpl::ModifyQos( grpc::ServerContext *context, const crane::grpc::ModifyQosRequest *request, crane::grpc::ModifyQosReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } - auto modify_res = - g_account_manager->ModifyQos(request->uid(), request->name(), - request->modify_field(), request->value()); + uint32_t uid = extract_result.value(); + + auto modify_res = g_account_manager->ModifyQos( + uid, request->name(), request->modify_field(), request->value()); if (modify_res) { response->set_ok(true); @@ -474,11 +535,18 @@ grpc::Status CraneCtldServiceImpl::QueryAccountInfo( grpc::ServerContext *context, const crane::grpc::QueryAccountInfoRequest *request, crane::grpc::QueryAccountInfoReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); std::unordered_map res_account_map; - auto modify_res = g_account_manager->QueryAccountInfo( - request->uid(), request->name(), &res_account_map); + auto modify_res = g_account_manager->QueryAccountInfo(uid, request->name(), + &res_account_map); if (modify_res) { response->set_ok(true); } else { @@ -529,11 +597,18 @@ grpc::Status CraneCtldServiceImpl::QueryUserInfo( grpc::ServerContext *context, const crane::grpc::QueryUserInfoRequest *request, crane::grpc::QueryUserInfoReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); std::unordered_map res_user_map; - auto modify_res = g_account_manager->QueryUserInfo( - request->uid(), request->name(), &res_user_map); + auto modify_res = + g_account_manager->QueryUserInfo(uid, request->name(), &res_user_map); if (modify_res) { response->set_ok(true); } else { @@ -588,10 +663,17 @@ grpc::Status CraneCtldServiceImpl::QueryQosInfo( crane::grpc::QueryQosInfoReply *response) { std::unordered_map res_qos_map; - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } - auto modify_res = g_account_manager->QueryQosInfo( - request->uid(), request->name(), &res_qos_map); + uint32_t uid = extract_result.value(); + + auto modify_res = + g_account_manager->QueryQosInfo(uid, request->name(), &res_qos_map); if (modify_res) { response->set_ok(true); } else { @@ -618,9 +700,16 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( grpc::ServerContext *context, const crane::grpc::DeleteAccountRequest *request, crane::grpc::DeleteAccountReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } - auto res = g_account_manager->DeleteAccount(request->uid(), request->name()); + uint32_t uid = extract_result.value(); + + auto res = g_account_manager->DeleteAccount(uid, request->name()); if (res) { response->set_ok(true); } else { @@ -633,9 +722,16 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( grpc::Status CraneCtldServiceImpl::DeleteUser( grpc::ServerContext *context, const crane::grpc::DeleteUserRequest *request, crane::grpc::DeleteUserReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); - auto res = g_account_manager->DeleteUser(request->uid(), request->name(), - request->account()); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); + auto res = + g_account_manager->DeleteUser(uid, request->name(), request->account()); if (res) { response->set_ok(true); } else { @@ -649,9 +745,16 @@ grpc::Status CraneCtldServiceImpl::DeleteUser( grpc::Status CraneCtldServiceImpl::DeleteQos( grpc::ServerContext *context, const crane::grpc::DeleteQosRequest *request, crane::grpc::DeleteQosReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); - auto res = g_account_manager->DeleteQos(request->uid(), request->name()); + auto res = g_account_manager->DeleteQos(uid, request->name()); if (res) { response->set_ok(true); } else { @@ -666,17 +769,24 @@ grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( grpc::ServerContext *context, const crane::grpc::BlockAccountOrUserRequest *request, crane::grpc::BlockAccountOrUserReply *response) { - // uint32_t uid = ExtractUIDFromMetadata(context); + auto extract_result = ExtractUIDFromCert(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); AccountManager::CraneExpected res; switch (request->entity_type()) { case crane::grpc::Account: - res = g_account_manager->BlockAccount(request->uid(), request->name(), - request->block()); + res = + g_account_manager->BlockAccount(uid, request->name(), request->block()); break; case crane::grpc::User: - res = g_account_manager->BlockUser(request->uid(), request->name(), - request->account(), request->block()); + res = g_account_manager->BlockUser(uid, request->name(), request->account(), + request->block()); break; default: std::unreachable(); @@ -696,6 +806,7 @@ grpc::Status CraneCtldServiceImpl::QueryClusterInfo( grpc::ServerContext *context, const crane::grpc::QueryClusterInfoRequest *request, crane::grpc::QueryClusterInfoReply *response) { + auto uid = ExtractUIDFromCert(context); *response = g_meta_container->QueryClusterInfo(*request); return grpc::Status::OK; } From 1f23cc0936e9a40ad7ef11232ef4279f0a6258a3 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:10:25 +0800 Subject: [PATCH 37/62] feat: delete jwt --- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 63 +++++++++++-------- src/CraneCtld/RpcService/CtldGrpcServer.h | 3 + src/Utilities/PublicHeader/CMakeLists.txt | 4 +- src/Utilities/PublicHeader/GrpcHelper.cpp | 59 ----------------- src/Utilities/PublicHeader/Jwt.cpp | 36 ----------- src/Utilities/PublicHeader/String.cpp | 41 ++++++++++-- .../PublicHeader/include/crane/GrpcHelper.h | 45 ------------- .../PublicHeader/include/crane/Jwt.h | 20 ------ .../PublicHeader/include/crane/String.h | 5 +- src/Utilities/VaultClient/VaultClient.cpp | 1 + 10 files changed, 83 insertions(+), 194 deletions(-) delete mode 100644 src/Utilities/PublicHeader/Jwt.cpp delete mode 100644 src/Utilities/PublicHeader/include/crane/Jwt.h diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index fe764508e..ecc824f7c 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -25,6 +25,7 @@ #include "CranedKeeper.h" #include "crane/GrpcHelper.h" #include "crane/String.h" +#include "crane/VaultClient.h" namespace Ctld { @@ -120,7 +121,7 @@ grpc::Status CraneCtldServiceImpl::ModifyTask( grpc::ServerContext *context, const crane::grpc::ModifyTaskRequest *request, crane::grpc::ModifyTaskReply *response) { using ModifyTaskRequest = crane::grpc::ModifyTaskRequest; - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { for (auto task_id : request->task_ids()) { response->add_not_modified_tasks(task_id); @@ -213,7 +214,7 @@ grpc::Status CraneCtldServiceImpl::ModifyNode( grpc::ServerContext *context, const crane::grpc::ModifyCranedStateRequest *request, crane::grpc::ModifyCranedStateReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { for (auto crane_id : request->craned_ids()) { response->add_not_modified_nodes(crane_id); @@ -286,7 +287,7 @@ grpc::Status CraneCtldServiceImpl::QueryTasksInfo( grpc::Status CraneCtldServiceImpl::AddAccount( grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -323,7 +324,7 @@ grpc::Status CraneCtldServiceImpl::AddAccount( grpc::Status CraneCtldServiceImpl::AddUser( grpc::ServerContext *context, const crane::grpc::AddUserRequest *request, crane::grpc::AddUserReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -373,7 +374,7 @@ grpc::Status CraneCtldServiceImpl::AddUser( grpc::Status CraneCtldServiceImpl::AddQos( grpc::ServerContext *context, const crane::grpc::AddQosRequest *request, crane::grpc::AddQosReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -415,7 +416,7 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( grpc::ServerContext *context, const crane::grpc::ModifyAccountRequest *request, crane::grpc::ModifyAccountReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -441,7 +442,7 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( grpc::Status CraneCtldServiceImpl::ModifyUser( grpc::ServerContext *context, const crane::grpc::ModifyUserRequest *request, crane::grpc::ModifyUserReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -509,7 +510,7 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( grpc::Status CraneCtldServiceImpl::ModifyQos( grpc::ServerContext *context, const crane::grpc::ModifyQosRequest *request, crane::grpc::ModifyQosReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -535,7 +536,7 @@ grpc::Status CraneCtldServiceImpl::QueryAccountInfo( grpc::ServerContext *context, const crane::grpc::QueryAccountInfoRequest *request, crane::grpc::QueryAccountInfoReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -597,7 +598,7 @@ grpc::Status CraneCtldServiceImpl::QueryUserInfo( grpc::ServerContext *context, const crane::grpc::QueryUserInfoRequest *request, crane::grpc::QueryUserInfoReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -663,7 +664,7 @@ grpc::Status CraneCtldServiceImpl::QueryQosInfo( crane::grpc::QueryQosInfoReply *response) { std::unordered_map res_qos_map; - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -700,7 +701,7 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( grpc::ServerContext *context, const crane::grpc::DeleteAccountRequest *request, crane::grpc::DeleteAccountReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -722,7 +723,7 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( grpc::Status CraneCtldServiceImpl::DeleteUser( grpc::ServerContext *context, const crane::grpc::DeleteUserRequest *request, crane::grpc::DeleteUserReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -745,7 +746,7 @@ grpc::Status CraneCtldServiceImpl::DeleteUser( grpc::Status CraneCtldServiceImpl::DeleteQos( grpc::ServerContext *context, const crane::grpc::DeleteQosRequest *request, crane::grpc::DeleteQosReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -769,7 +770,7 @@ grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( grpc::ServerContext *context, const crane::grpc::BlockAccountOrUserRequest *request, crane::grpc::BlockAccountOrUserReply *response) { - auto extract_result = ExtractUIDFromCert(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); @@ -806,11 +807,31 @@ grpc::Status CraneCtldServiceImpl::QueryClusterInfo( grpc::ServerContext *context, const crane::grpc::QueryClusterInfoRequest *request, crane::grpc::QueryClusterInfoReply *response) { - auto uid = ExtractUIDFromCert(context); + auto uid = CheckCertAllowedAndExtractUIDFromCert_(context); *response = g_meta_container->QueryClusterInfo(*request); return grpc::Status::OK; } +std::expected +CraneCtldServiceImpl::CheckCertAllowedAndExtractUIDFromCert_( + const grpc::ServerContext *context) { + auto cert = context->auth_context()->FindPropertyValues("x509_pem_cert"); + if (cert.empty()) return std::unexpected(false); + + std::string certificate = std::string(cert[0].data(), cert[0].size()); + + auto result = util::ParseCertificate(certificate); + if (!result) return std::unexpected(false); + + if (!g_vault_client->IsCertAllowed(result.value().second)) return false; + + std::vector cn_parts = absl::StrSplit(result.value().first, '.'); + if (cn_parts.size() != 3 || cn_parts[0].empty()) + return std::unexpected(false); + + return static_cast(std::stoul(cn_parts[0])); +} + CtldServer::CtldServer() : m_service_impl_(nullptr) { m_service_impl_ = std::make_unique(this); @@ -824,16 +845,6 @@ CtldServer::CtldServer() : m_service_impl_(nullptr) { &builder, cranectld_listen_addr, listen_conf.CraneCtldListenPort, vault_conf.ExternalCerts, vault_conf.ExternalCACerts.CACertContent); - // std::vector< - // std::unique_ptr> - // creators; - // creators.push_back( - // std::unique_ptr( - // new - // JwtAuthInterceptorFactory(g_config.ListenConf.JwtSecretContent))); - - // builder.experimental().SetInterceptorCreators(std::move(creators)); - builder.RegisterService(m_service_impl_.get()); m_server_ = builder.BuildAndStart(); diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.h b/src/CraneCtld/RpcService/CtldGrpcServer.h index 892194c23..a6d9dcf84 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.h +++ b/src/CraneCtld/RpcService/CtldGrpcServer.h @@ -141,6 +141,9 @@ class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { crane::grpc::QueryClusterInfoReply *response) override; private: + std::expected CheckCertAllowedAndExtractUIDFromCert_( + const grpc::ServerContext *context); + CtldServer *m_ctld_server_; }; diff --git a/src/Utilities/PublicHeader/CMakeLists.txt b/src/Utilities/PublicHeader/CMakeLists.txt index d62e7cfb7..decb8e5e3 100644 --- a/src/Utilities/PublicHeader/CMakeLists.txt +++ b/src/Utilities/PublicHeader/CMakeLists.txt @@ -1,5 +1,5 @@ add_library(Utility_PublicHeader - String.cpp Network.cpp OS.cpp PublicHeader.cpp Logger.cpp Jwt.cpp + String.cpp Network.cpp OS.cpp PublicHeader.cpp Logger.cpp include/crane/String.h include/crane/Network.h include/crane/OS.h @@ -7,7 +7,6 @@ add_library(Utility_PublicHeader include/crane/Lock.h include/crane/Pointer.h include/crane/Logger.h - include/crane/Jwt.h include/crane/PasswordEntry.h include/crane/AtomicHashMap.h GrpcHelper.cpp @@ -17,7 +16,6 @@ target_link_libraries(Utility_PublicHeader PUBLIC spdlog::spdlog crane_proto_lib fpm - jwt-cpp::jwt-cpp ) # This trimmed version is used for PAM module diff --git a/src/Utilities/PublicHeader/GrpcHelper.cpp b/src/Utilities/PublicHeader/GrpcHelper.cpp index 1fc5fc281..0c6b73932 100644 --- a/src/Utilities/PublicHeader/GrpcHelper.cpp +++ b/src/Utilities/PublicHeader/GrpcHelper.cpp @@ -18,47 +18,6 @@ #include "crane/GrpcHelper.h" -// grpc::Status JwtAuthProcessor::Process(const InputMetadata& auth_metadata, -// grpc::AuthContext* context, -// OutputMetadata* -// consumed_auth_metadata, -// OutputMetadata* response_metadata) { -// auto iter = auth_metadata.find("Authorization"); -// if (iter == auth_metadata.end()) { -// return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Miss token"); -// } - -// auto token = iter->second.data(); -// if (!util::VerifyToken(jwt_secret_, token)) { -// return grpc::Status(grpc::StatusCode::UNAUTHENTICATED, "Invalid token"); -// } - -// return grpc::Status::OK; -// } - -void JwtAuthInterceptor::Intercept( - grpc::experimental::InterceptorBatchMethods* methods) { - if (methods->QueryInterceptionHookPoint( - grpc::experimental::InterceptionHookPoints:: - POST_RECV_INITIAL_METADATA)) { - if (strcmp(info_->method(), "/crane.grpc.CraneCtld/Login") != 0) { - auto* metadata_map = methods->GetRecvInitialMetadata(); - - auto iter = metadata_map->find("authorization"); - if (iter == metadata_map->end()) { - info_->server_context()->TryCancel(); - return; - } - std::string token(iter->second.data(), iter->second.size()); - if (!util::VerifyToken(jwt_secret_, token)) { - info_->server_context()->TryCancel(); - return; - } - } - } - methods->Proceed(); -} - static std::string GrpcFormatIpAddress(std::string const& addr) { // Grpc needs to use [] to wrap ipv6 address if (int ip_ver = crane::GetIpAddrVer(addr); ip_ver == 6) @@ -67,24 +26,6 @@ static std::string GrpcFormatIpAddress(std::string const& addr) { return addr; } -// Retrieve UID from the certificate. -std::expected ExtractUIDFromCert( - const grpc::ServerContext* context) { - auto cert = context->auth_context()->FindPropertyValues("x509_pem_cert"); - if (cert.empty()) return std::unexpected(false); - - std::string certificate = std::string(cert[0].data(), cert[0].size()); - - auto result = util::ParseCertificate(certificate); - if (!result) return std::unexpected(false); - - std::vector cn_parts = absl::StrSplit(result.value(), '.'); - if (cn_parts.size() != 3 || cn_parts[0].empty()) - return std::unexpected(false); - - return static_cast(std::stoul(cn_parts[0])); -} - void ServerBuilderSetCompression(grpc::ServerBuilder* builder) { builder->SetDefaultCompressionAlgorithm(GRPC_COMPRESS_GZIP); } diff --git a/src/Utilities/PublicHeader/Jwt.cpp b/src/Utilities/PublicHeader/Jwt.cpp deleted file mode 100644 index de217dc49..000000000 --- a/src/Utilities/PublicHeader/Jwt.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "crane/Jwt.h" - -namespace util { -std::string GenerateToken( - const std::string& secret, - const std::unordered_map& claims) { - auto creater = jwt::create().set_issuer("crane").set_type("JWS"); - for (const auto& [k, v] : claims) { - creater.set_payload_claim(k, jwt::claim(v)); - } - return creater.sign(jwt::algorithm::hs256{secret}); -} - -bool VerifyToken(const std::string& secret, const std::string& token) { - try { - auto decoded = jwt::decode(token); - jwt::verify() - .allow_algorithm(jwt::algorithm::hs256{secret}) - .with_issuer("crane") - .verify(decoded); - } catch (std::invalid_argument& a) { - return false; - } catch (std::runtime_error& e) { - return false; - } - - return true; -} - -std::string GetClaim(const std::string& key, const std::string& token) { - auto decoded = jwt::decode(token); - - return decoded.get_payload_claim(key).as_string(); -} - -} // namespace util diff --git a/src/Utilities/PublicHeader/String.cpp b/src/Utilities/PublicHeader/String.cpp index 91452d509..74d25c8d3 100644 --- a/src/Utilities/PublicHeader/String.cpp +++ b/src/Utilities/PublicHeader/String.cpp @@ -411,7 +411,7 @@ std::string GenerateCommaSeparatedString(const int val) { return absl::StrJoin(val_vec, ","); } -std::expected ParseCertificate(const std::string &cert_pem) { +std::expected ParseCertificate(const std::string &cert_pem) { // Load the certificate content into a BIO (memory buffer). BIO *bio = BIO_new_mem_buf(cert_pem.data(), cert_pem.size()); if (!bio) return std::unexpected(false); @@ -425,17 +425,50 @@ std::expected ParseCertificate(const std::string &cert_pem) { // Retrieve Subject information. X509_NAME *subject = X509_get_subject_name(cert); - if (!subject) return std::unexpected(false); + if (!subject) { + X509_free(cert); + BIO_free(bio); + return std::unexpected(false); + } char cn[256]; int len = X509_NAME_get_text_by_NID(subject, NID_commonName, cn, sizeof(cn)); - if (len <= 0) return std::unexpected(false); + if (len <= 0) { + X509_free(cert); + BIO_free(bio); + return std::unexpected(false); + } + + ASN1_INTEGER *serial = X509_get_serialNumber(cert); + if (!serial) { + X509_free(cert); + BIO_free(bio); + return std::unexpected(false); + } + + BIGNUM *bn = ASN1_INTEGER_to_BN(serial, nullptr); + if (!bn) { + X509_free(cert); + BIO_free(bio); + return std::unexpected(false); + } + + char *hex = BN_bn2hex(bn); + if (!hex) { + BN_free(bn); + X509_free(cert); + BIO_free(bio); + return std::unexpected(false); + } + std::string serial_number = std::string(hex); // free the memory + OPENSSL_free(hex); + BN_free(bn); X509_free(cert); BIO_free(bio); - return cn; + return CertPair{cn, serial_number}; } } // namespace util diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index cd7cabdc6..cb0475303 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -24,11 +24,7 @@ #include #include -#include - -#include "crane/Jwt.h" #include "crane/Network.h" -#include "crane/String.h" struct ServerCertificateConfig { std::string ServerCertFilePath; @@ -47,47 +43,6 @@ struct CACertificateConfig { std::string CACertContent; }; -// class JwtAuthProcessor : public grpc::AuthMetadataProcessor { -// public: -// JwtAuthProcessor(std::string secret) : jwt_secret_(secret) {} -// grpc::Status Process(const InputMetadata& auth_metadata, -// grpc::AuthContext* context, -// OutputMetadata* consumed_auth_metadata, -// OutputMetadata* response_metadata) override; - -// private: -// std::string jwt_secret_; -// }; - -class JwtAuthInterceptor : public grpc::experimental::Interceptor { - public: - explicit JwtAuthInterceptor(grpc::experimental::ServerRpcInfo* info, - std::string secret) - : info_(info), jwt_secret_(secret) {} - - void Intercept(grpc::experimental::InterceptorBatchMethods* methods) override; - - private: - grpc::experimental::ServerRpcInfo* info_; - std::string jwt_secret_; -}; - -class JwtAuthInterceptorFactory - : public grpc::experimental::ServerInterceptorFactoryInterface { - public: - JwtAuthInterceptorFactory(std::string secret) : jwt_secret_(secret) {} - grpc::experimental::Interceptor* CreateServerInterceptor( - grpc::experimental::ServerRpcInfo* info) override { - return new JwtAuthInterceptor(info, jwt_secret_); - } - - private: - std::string jwt_secret_; -}; - -std::expected ExtractUIDFromCert( - const grpc::ServerContext* context); - void ServerBuilderSetCompression(grpc::ServerBuilder* builder); void ServerBuilderSetKeepAliveArgs(grpc::ServerBuilder* builder); diff --git a/src/Utilities/PublicHeader/include/crane/Jwt.h b/src/Utilities/PublicHeader/include/crane/Jwt.h deleted file mode 100644 index b298ae04b..000000000 --- a/src/Utilities/PublicHeader/include/crane/Jwt.h +++ /dev/null @@ -1,20 +0,0 @@ - - -#pragma once - -#include - -#include -#include - -namespace util { - -std::string GenerateToken( - const std::string& secret, - const std::unordered_map& claims); - -bool VerifyToken(const std::string& secret, const std::string& token); - -std::string GetClaim(const std::string& key, const std::string& token); - -} // namespace util \ No newline at end of file diff --git a/src/Utilities/PublicHeader/include/crane/String.h b/src/Utilities/PublicHeader/include/crane/String.h index 2cc8f581f..a2a07bbe7 100644 --- a/src/Utilities/PublicHeader/include/crane/String.h +++ b/src/Utilities/PublicHeader/include/crane/String.h @@ -38,6 +38,9 @@ namespace util { +using CertPair = std::pair; // serial number + std::string ReadFileIntoString(std::filesystem::path const &p); std::string ReadableMemory(uint64_t memory_bytes); @@ -85,6 +88,6 @@ std::string ReadableGrpcDresInNode( std::string GenerateCommaSeparatedString(const int val); -std::expected ParseCertificate(const std::string &cert_pem); +std::expected ParseCertificate(const std::string &cert_pem); } // namespace util \ No newline at end of file diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index f23692f9f..d1921576c 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -169,6 +169,7 @@ bool VaultClient::RevokeCert(const std::string& serial_number) { } bool VaultClient::IsCertAllowed(const std::string& serial_number) { + // TODO: 加锁 if (allowed_certs_.contains(serial_number)) return true; try { From cf97f9833d9389b2b0bde0bbb327eae5adf25fda Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:14:54 +0800 Subject: [PATCH 38/62] refactor --- dependencies/cmake/CMakeLists.txt | 2 +- .../{jwt-cpp => nlohmann_json}/CMakeLists.txt | 15 +-------------- 2 files changed, 2 insertions(+), 15 deletions(-) rename dependencies/cmake/{jwt-cpp => nlohmann_json}/CMakeLists.txt (57%) diff --git a/dependencies/cmake/CMakeLists.txt b/dependencies/cmake/CMakeLists.txt index 49b1d2b35..39b382c49 100644 --- a/dependencies/cmake/CMakeLists.txt +++ b/dependencies/cmake/CMakeLists.txt @@ -13,7 +13,7 @@ if(CRANE_USE_MIMALLOC) add_subdirectory(mimalloc) endif() add_subdirectory(BSThreadPool) -add_subdirectory(jwt-cpp) +add_subdirectory(nlohmann_json) add_subdirectory(yaml-cpp) add_subdirectory(fmt) add_subdirectory(googletest) diff --git a/dependencies/cmake/jwt-cpp/CMakeLists.txt b/dependencies/cmake/nlohmann_json/CMakeLists.txt similarity index 57% rename from dependencies/cmake/jwt-cpp/CMakeLists.txt rename to dependencies/cmake/nlohmann_json/CMakeLists.txt index 058e88739..853347711 100644 --- a/dependencies/cmake/jwt-cpp/CMakeLists.txt +++ b/dependencies/cmake/nlohmann_json/CMakeLists.txt @@ -6,8 +6,6 @@ else () set(JSON_SRC_URL "https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.tar.gz") endif () -set(JWT_CPP_SRC_URL "https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz") - FetchContent_Declare(json URL ${JSON_SRC_URL} URL_HASH SHA256=d69f9deb6a75e2580465c6c4c5111b89c4dc2fa94e3a85fcd2ffcd9a143d9273 @@ -18,15 +16,4 @@ FetchContent_GetProperties(json) if (NOT json_POPULATED) FetchContent_Populate(json) add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR} EXCLUDE_FROM_ALL) -endif () - -set(nlohmann_json_DIR "${json_BINARY_DIR}") - -fetchcontent_declare(jwt-cpp - URL ${JWT_CPP_SRC_URL} - URL_HASH SHA256=b9eb270e3ba8221e4b2bc38723c9a1cb4fa6c241a42908b9a334daff31137406 - INACTIVITY_TIMEOUT 5 - ) -set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "disable building examples" FORCE) - -fetchcontent_makeavailable(jwt-cpp) \ No newline at end of file +endif () \ No newline at end of file From f70098d6cfb1d0d855ca060b75adf33ee7746ab8 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 2 Jan 2025 10:31:49 +0800 Subject: [PATCH 39/62] feat: task interface use uid from cert --- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 61 +++++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index ecc824f7c..52fe72bd4 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -33,6 +33,20 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTask( grpc::ServerContext *context, const crane::grpc::SubmitBatchTaskRequest *request, crane::grpc::SubmitBatchTaskReply *response) { + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason("Permission Denied."); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); + if (uid != request->task().uid()) { + response->set_ok(false); + response->set_reason("Identity mismatch"); + return grpc::Status::OK; + } + auto task = std::make_unique(); task->SetFieldsByTaskToCtld(request->task()); @@ -66,10 +80,23 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTasks( const auto &task_to_ctld = request->task(); results.reserve(task_count); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); + if (!extract_result) { + for (int i = 0; i < task_count; i++) { + response->mutable_reason_list()->Add("Permission Denied."); + } + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); + for (int i = 0; i < task_count; i++) { + if (uid != task_to_ctld.uid()) { + results.emplace_back(std::unexpected("Identity mismatch")); + continue; + } auto task = std::make_unique(); task->SetFieldsByTaskToCtld(task_to_ctld); - auto result = g_task_scheduler->SubmitTaskToScheduler(std::move(task)); results.emplace_back(std::move(result)); } @@ -87,6 +114,24 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTasks( grpc::Status CraneCtldServiceImpl::CancelTask( grpc::ServerContext *context, const crane::grpc::CancelTaskRequest *request, crane::grpc::CancelTaskReply *response) { + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); + if (!extract_result) { + for (auto task_id : request->filter_task_ids()) { + response->add_not_cancelled_tasks(task_id); + response->add_not_cancelled_reasons("Permission Denied."); + } + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); + if (uid != request->operator_uid()) { + for (auto task_id : request->filter_task_ids()) { + response->add_not_cancelled_tasks(task_id); + response->add_not_cancelled_reasons("Permission Denied."); + } + return grpc::Status::OK; + } + *response = g_task_scheduler->CancelPendingOrRunningTask(*request); return grpc::Status::OK; } @@ -95,6 +140,9 @@ grpc::Status CraneCtldServiceImpl::QueryCranedInfo( grpc::ServerContext *context, const crane::grpc::QueryCranedInfoRequest *request, crane::grpc::QueryCranedInfoReply *response) { + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); + if (!extract_result) return grpc::Status::OK; + if (request->craned_name().empty()) { *response = g_meta_container->QueryAllCranedInfo(); } else { @@ -108,6 +156,9 @@ grpc::Status CraneCtldServiceImpl::QueryPartitionInfo( grpc::ServerContext *context, const crane::grpc::QueryPartitionInfoRequest *request, crane::grpc::QueryPartitionInfoReply *response) { + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); + if (!extract_result) return grpc::Status::OK; + if (request->partition_name().empty()) { *response = g_meta_container->QueryAllPartitionInfo(); } else { @@ -125,7 +176,7 @@ grpc::Status CraneCtldServiceImpl::ModifyTask( if (!extract_result) { for (auto task_id : request->task_ids()) { response->add_not_modified_tasks(task_id); - response->add_not_modified_reasons("uid is invalid."); + response->add_not_modified_reasons("Permission Denied."); } return grpc::Status::OK; } @@ -218,7 +269,7 @@ grpc::Status CraneCtldServiceImpl::ModifyNode( if (!extract_result) { for (auto crane_id : request->craned_ids()) { response->add_not_modified_nodes(crane_id); - response->add_not_modified_reasons("uid is invalid."); + response->add_not_modified_reasons("Permission Denied."); } return grpc::Status::OK; } @@ -807,7 +858,9 @@ grpc::Status CraneCtldServiceImpl::QueryClusterInfo( grpc::ServerContext *context, const crane::grpc::QueryClusterInfoRequest *request, crane::grpc::QueryClusterInfoReply *response) { - auto uid = CheckCertAllowedAndExtractUIDFromCert_(context); + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); + if (!extract_result) return grpc::Status::OK; + *response = g_meta_container->QueryClusterInfo(*request); return grpc::Status::OK; } From aac7cee6afb26909f1c33a2b372e38d7b9bf6013 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 2 Jan 2025 13:42:03 +0800 Subject: [PATCH 40/62] feat: Login node restriction --- etc/config.yaml | 1 + protos/PublicDefs.proto | 12 +++---- src/CraneCtld/CraneCtld.cpp | 12 +++++++ src/CraneCtld/CtldPublicDefs.h | 1 + .../RpcService/CtldForCforedServer.cpp | 35 +++++++++++++++++++ 5 files changed, 54 insertions(+), 7 deletions(-) diff --git a/etc/config.yaml b/etc/config.yaml index 73c5736f4..52d16a59a 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -27,6 +27,7 @@ Vault: Port: 8200 Token: REMOVED # Token cannot be leaked DomainSuffix: crane.com + Nodes: "cn[15-18]" ExternalCertFilePath: /etc/crane/tls/external.pem ExternalKeyFilePath: /etc/crane/tls/external.key diff --git a/protos/PublicDefs.proto b/protos/PublicDefs.proto index 8a089c6fc..5a7ce2900 100644 --- a/protos/PublicDefs.proto +++ b/protos/PublicDefs.proto @@ -342,7 +342,6 @@ enum ErrCode { ERR_INVALID_ADMIN_LEVEL = 10007; // Invalid permission level ERR_USER_ACCOUNT_MISMATCH = 10008; // User does not belong to the account ERR_NO_ACCOUNT_SPECIFIED = 10009; - ERR_PASSWORD_MISMATCH = 100010; ERR_INVALID_ACCOUNT = 10011; ERR_DUPLICATE_ACCOUNT = 10012; @@ -465,12 +464,11 @@ message UserInfo { uint32 uid = 1; string name = 2; - string password = 3; - string account = 4; - bool blocked = 5; - repeated AllowedPartitionQos allowed_partition_qos_list = 6; - repeated string coordinator_accounts = 7; - AdminLevel admin_level = 8; + string account = 3; + bool blocked = 4; + repeated AllowedPartitionQos allowed_partition_qos_list = 5; + repeated string coordinator_accounts = 6; + AdminLevel admin_level = 7; } message QosInfo { diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index bda5977a7..537d593f0 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -312,6 +312,18 @@ void ParseConfig(int argc, char** argv) { g_config.VaultConf.DomainSuffix = vault_config["DomainSuffix"].as(); + if (vault_config["Nodes"]) { + std::string nodes = vault_config["Nodes"].as(); + std::list name_list; + if (!util::ParseHostList(absl::StripAsciiWhitespace(nodes).data(), + &name_list)) { + CRANE_ERROR("Illegal login node name string format."); + std::exit(1); + } + g_config.VaultConf.AllowedNodes = std::unordered_set( + name_list.begin(), name_list.end()); + } + if (vault_config["ExternalCertFilePath"]) { external_certs.ServerCertFilePath = vault_config["ExternalCertFilePath"].as(); diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 5a30e7689..ae90a705e 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -114,6 +114,7 @@ struct Config { std::string Port; std::string Token; std::string DomainSuffix; + std::unordered_set AllowedNodes; ServerCertificateConfig ExternalCerts; CACertificateConfig ExternalCACerts; diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index 523dbcce9..d791936c0 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -25,6 +25,7 @@ #include "../TaskScheduler.h" #include "CranedKeeper.h" #include "CtldGrpcServer.h" +#include "crane/Network.h" #include "crane/String.h" namespace Ctld { @@ -212,6 +213,40 @@ grpc::Status CtldForCforedServiceImpl::SignUserCertificate( grpc::ServerContext *context, const crane::grpc::SignUserCertificateRequest *request, crane::grpc::SignUserCertificateResponse *response) { + if (!g_config.VaultConf.AllowedNodes.empty()) { + std::string client_address = context->peer(); + std::vector str_list = absl::StrSplit(client_address, ":"); + std::string hostname; + bool resolve_result = false; + if (str_list[0] == "ipv4") { + ipv4_t addr; + if (!crane::StrToIpv4(str_list[1], &addr)) { + CRANE_ERROR("Failed to parse ipv4 address: {}", str_list[1]); + } else { + resolve_result = crane::ResolveHostnameFromIpv4(addr, &hostname); + } + } else { + ipv6_t addr; + if (!crane::StrToIpv6(str_list[1], &addr)) { + CRANE_ERROR("Failed to parse ipv6 address: {}", str_list[1]); + } else { + resolve_result = crane::ResolveHostnameFromIpv6(addr, &hostname); + } + } + + CRANE_INFO("Resolve hostname from address: {} -> {}", client_address, + hostname); + + if (!resolve_result || + (!g_config.VaultConf.AllowedNodes.contains(hostname) && + !g_config.VaultConf.AllowedNodes.contains( + hostname + g_config.VaultConf.DomainSuffix))) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + return grpc::Status::OK; + } + } + auto result = g_account_manager->SignUserCertificate( request->uid(), request->csr_content(), request->alt_names()); if (!result) { From 6838d73c300b234375cfecded1f54235586e7580 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Fri, 3 Jan 2025 09:29:52 +0800 Subject: [PATCH 41/62] fix: Login node restriction --- src/CraneCtld/RpcService/CtldForCforedServer.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index d791936c0..705aa98e4 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -236,11 +236,9 @@ grpc::Status CtldForCforedServiceImpl::SignUserCertificate( CRANE_INFO("Resolve hostname from address: {} -> {}", client_address, hostname); - + std::vector name_list = absl::StrSplit(hostname, "."); if (!resolve_result || - (!g_config.VaultConf.AllowedNodes.contains(hostname) && - !g_config.VaultConf.AllowedNodes.contains( - hostname + g_config.VaultConf.DomainSuffix))) { + !g_config.VaultConf.AllowedNodes.contains(name_list[0])) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); return grpc::Status::OK; From c29ec62778ad77b58a416556da415a6050951749 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Fri, 3 Jan 2025 11:23:52 +0800 Subject: [PATCH 42/62] feat: revoke user Credential --- protos/Crane.proto | 11 +++++ protos/PublicDefs.proto | 1 + src/CraneCtld/AccountManager.cpp | 39 +++++++++++++++ src/CraneCtld/AccountManager.h | 3 ++ src/CraneCtld/RpcService/CtldGrpcServer.cpp | 40 ++++++++++++--- src/CraneCtld/RpcService/CtldGrpcServer.h | 5 ++ src/Utilities/PublicHeader/String.cpp | 8 +++ src/Utilities/VaultClient/VaultClient.cpp | 49 ++++++++++++++++--- .../VaultClient/include/crane/VaultClient.h | 17 ++++++- 9 files changed, 156 insertions(+), 17 deletions(-) diff --git a/protos/Crane.proto b/protos/Crane.proto index cb5ecd799..1a31d1797 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -758,6 +758,15 @@ message SignUserCertificateResponse { ErrCode reason = 4; } +message ResetUserCredentialRequest { + string username = 1; +} + +message ResetUserCredentialReply { + bool ok = 1; + ErrCode reason = 2; +} + // Todo: Divide service into two parts: one for Craned and one for Crun // We need to distinguish the message sender // and have some kind of authentication @@ -794,6 +803,8 @@ service CraneCtld { rpc BlockAccountOrUser(BlockAccountOrUserRequest) returns (BlockAccountOrUserReply); + rpc ResetUserCredential(ResetUserCredentialRequest) returns (ResetUserCredentialReply); + /* RPCs called from cinfo */ rpc QueryClusterInfo(QueryClusterInfoRequest) returns (QueryClusterInfoReply); diff --git a/protos/PublicDefs.proto b/protos/PublicDefs.proto index 5a7ce2900..e86d822ac 100644 --- a/protos/PublicDefs.proto +++ b/protos/PublicDefs.proto @@ -395,6 +395,7 @@ enum ErrCode { ERR_SIGN_CERTIFICATE = 10119; ERR_DUPLICATE_CERTIFICATE = 10120; + ERR_REVOKE_CERTIFICATE = 10121; } enum EntityType { diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 4f5cd6f71..22cabcd7d 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1067,6 +1067,45 @@ AccountManager::CraneExpected AccountManager::SignUserCertificate( return sign_response->certificate; } +AccountManager::CraneExpected AccountManager::ResetUserCertificate( + uint32_t uid, const std::string& name) { + util::write_lock_guard user_guard(m_rw_user_mutex_); + CraneExpected result{}; + + auto user_result = GetUserInfoByUidNoLock_(uid); + if (!user_result) return std::unexpected(user_result.error()); + const User* op_user = user_result.value(); + + const User* user = GetExistedUserInfoNoLock_(name); + if (!user) return std::unexpected(CraneErrCode::ERR_INVALID_USER); + + if (!CheckIfUserHasHigherPrivThan_(*op_user, User::None)) + return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); + + if (op_user->name != name && + !CheckIfUserHasHigherPrivThan_(*op_user, user->admin_level)) + return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); + + if (!g_vault_client->RevokeCert(user->serial_number)) + return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); + + // Save the serial number in the database. + mongocxx::client_session::with_transaction_cb callback = + [&](mongocxx::client_session* session) { + g_db_client->UpdateEntityOne(MongodbClient::EntityType::USER, "$set", + name, "serial_number", ""); + }; + + // Update to database + if (!g_db_client->CommitTransaction(callback)) { + return std::unexpected(CraneErrCode::ERR_UPDATE_DATABASE); + } + + m_user_map_[name]->serial_number = ""; + + return result; +} + /****************************************** * NOLOCK ******************************************/ diff --git a/src/CraneCtld/AccountManager.h b/src/CraneCtld/AccountManager.h index e58f64a54..837a48b38 100644 --- a/src/CraneCtld/AccountManager.h +++ b/src/CraneCtld/AccountManager.h @@ -149,6 +149,9 @@ class AccountManager { const std::string& csr_content, const std::string& alt_names); + CraneExpected ResetUserCertificate(uint32_t uid, + const std::string& name); + private: void InitDataMap_(); diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index 52fe72bd4..837edd9aa 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -652,7 +652,7 @@ grpc::Status CraneCtldServiceImpl::QueryUserInfo( auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); return grpc::Status::OK; } @@ -718,7 +718,7 @@ grpc::Status CraneCtldServiceImpl::QueryQosInfo( auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); return grpc::Status::OK; } @@ -755,7 +755,7 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); return grpc::Status::OK; } @@ -777,7 +777,7 @@ grpc::Status CraneCtldServiceImpl::DeleteUser( auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); return grpc::Status::OK; } @@ -800,7 +800,7 @@ grpc::Status CraneCtldServiceImpl::DeleteQos( auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); return grpc::Status::OK; } @@ -824,7 +824,7 @@ grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); if (!extract_result) { response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); return grpc::Status::OK; } @@ -854,6 +854,31 @@ grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( return grpc::Status::OK; } +grpc::Status CraneCtldServiceImpl::ResetUserCredential( + grpc::ServerContext *context, + const crane::grpc::ResetUserCredentialRequest *request, + crane::grpc::ResetUserCredentialReply *response) { + auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); + if (!extract_result) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + return grpc::Status::OK; + } + + uint32_t uid = extract_result.value(); + + auto result = + g_account_manager->ResetUserCertificate(uid, request->username()); + if (!result) { + response->set_ok(false); + response->set_reason(result.error()); + } else { + response->set_ok(true); + } + + return grpc::Status::OK; +} + grpc::Status CraneCtldServiceImpl::QueryClusterInfo( grpc::ServerContext *context, const crane::grpc::QueryClusterInfoRequest *request, @@ -876,7 +901,8 @@ CraneCtldServiceImpl::CheckCertAllowedAndExtractUIDFromCert_( auto result = util::ParseCertificate(certificate); if (!result) return std::unexpected(false); - if (!g_vault_client->IsCertAllowed(result.value().second)) return false; + if (!g_vault_client->IsCertAllowed(result.value().second)) + return std::unexpected(false); std::vector cn_parts = absl::StrSplit(result.value().first, '.'); if (cn_parts.size() != 3 || cn_parts[0].empty()) diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.h b/src/CraneCtld/RpcService/CtldGrpcServer.h index a6d9dcf84..b32a19268 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.h +++ b/src/CraneCtld/RpcService/CtldGrpcServer.h @@ -135,6 +135,11 @@ class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { const crane::grpc::BlockAccountOrUserRequest *request, crane::grpc::BlockAccountOrUserReply *response) override; + grpc::Status ResetUserCredential( + grpc::ServerContext *context, + const crane::grpc::ResetUserCredentialRequest *request, + crane::grpc::ResetUserCredentialReply *response) override; + grpc::Status QueryClusterInfo( grpc::ServerContext *context, const crane::grpc::QueryClusterInfoRequest *request, diff --git a/src/Utilities/PublicHeader/String.cpp b/src/Utilities/PublicHeader/String.cpp index 74d25c8d3..979ed0be6 100644 --- a/src/Utilities/PublicHeader/String.cpp +++ b/src/Utilities/PublicHeader/String.cpp @@ -22,6 +22,8 @@ #include #include +#include + #include "crane/Logger.h" namespace util { @@ -461,6 +463,12 @@ std::expected ParseCertificate(const std::string &cert_pem) { return std::unexpected(false); } std::string serial_number = std::string(hex); + std::regex re("(.{2})"); + serial_number = std::regex_replace(serial_number, re, "$1:"); + serial_number.pop_back(); // remove the last colon + + std::transform(serial_number.begin(), serial_number.end(), + serial_number.begin(), ::tolower); // free the memory OPENSSL_free(hex); diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index d1921576c..7a76605c3 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -21,12 +21,13 @@ namespace vault { VaultClient::VaultClient(const std::string& root_token, - const std::string& address, const std::string& port) - : address_(address), port_(port) { + const std::string& address, const std::string& port, + bool tls) + : address_(address), port_(port), tls_(tls) { Vault::TokenStrategy tokenStrategy{Vault::Token{root_token}}; Vault::Config config = Vault::ConfigBuilder() .withDebug(false) - .withTlsEnabled(false) + .withTlsEnabled(tls_) .withHost(Vault::Host{address_}) .withPort(Vault::Port{port_}) .build(); @@ -163,22 +164,36 @@ bool VaultClient::RevokeCert(const std::string& serial_number) { return false; } - allowed_certs_.erase(serial_number); + { + util::write_lock_guard write_lock(rw_mutex_); + allowed_certs_.erase(serial_number); + } return true; } bool VaultClient::IsCertAllowed(const std::string& serial_number) { - // TODO: 加锁 - if (allowed_certs_.contains(serial_number)) return true; + { + util::read_lock_guard read_lock(rw_mutex_); + if (allowed_certs_.contains(serial_number)) return true; + } try { - // TODO: 检查crl列表 + auto response = ListRevokeCertificate_(); + if (!response) return false; + auto data = nlohmann::json::parse(response.value())["data"]; + for (const auto& key : data["keys"]) { + std::string key_str = key; + if (key_str == serial_number) return false; + } } catch (const std::exception& e) { return false; } - allowed_certs_.emplace(serial_number); + { + util::write_lock_guard write_lock(rw_mutex_); + allowed_certs_.emplace(serial_number); + } return true; } @@ -252,4 +267,22 @@ bool VaultClient::IssureExternalCert_(const std::string& role_name, return true; } +std::optional VaultClient::ListRevokeCertificate_() { + return Vault::HttpConsumer::list( + *root_client_, GetPkiUrl_(Vault::SecretMount{"pki_external"}, + Vault::Path{"certs/revoked"})); +} + +Vault::Url VaultClient::GetPkiUrl_(const Vault::SecretMount secret_mount, + const Vault::Path& path) const { + return GetUrl_("/v1/" + secret_mount, + path.empty() ? path : Vault::Path{"/" + path}); +} + +Vault::Url VaultClient::GetUrl_(const std::string& base, + const Vault::Path& path) const { + return Vault::Url{(tls_ ? "https://" : "http://") + address_ + ":" + port_ + + base + path}; +} + } // namespace vault \ No newline at end of file diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index b418d9ed1..3dd8046b6 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -27,6 +27,7 @@ #include #include "crane/GrpcHelper.h" +#include "crane/Lock.h" #include "crane/Logger.h" #include "crane/OS.h" @@ -42,7 +43,8 @@ using AllowedCerts = std::unordered_set; class VaultClient { public: VaultClient(const std::string& root_token, const std::string& address, - const std::string& port); + const std::string& port, bool tls = false); + bool InitPki(const std::string& domains, CACertificateConfig* external_ca, ServerCertificateConfig* external_cert); @@ -64,14 +66,25 @@ class VaultClient { const std::string& domains, ServerCertificateConfig* external_cert); + std::optional ListRevokeCertificate_(); + + Vault::Url GetUrl_(const std::string& base, const Vault::Path& path) const; + + Vault::Url GetPkiUrl_(const Vault::SecretMount secret_mount, + const Vault::Path& path) const; + std::unique_ptr root_client_; std::unique_ptr pki_root_; std::unique_ptr pki_internal_; std::unique_ptr pki_external_; - AllowedCerts allowed_certs_; + // TODO: 采用并行容器，提高性能 + AllowedCerts allowed_certs_ ABSL_GUARDED_BY(rw_mutex_); + util::rw_mutex rw_mutex_; + std::string address_; std::string port_; + bool tls_; }; } // namespace vault From f073134bb8c6bdc3af747e3f7c04f1dd066d9c82 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Fri, 3 Jan 2025 13:45:13 +0800 Subject: [PATCH 43/62] feat: vault health --- src/Utilities/VaultClient/VaultClient.cpp | 27 ++++++++++++++----- .../VaultClient/include/crane/VaultClient.h | 2 ++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index 7a76605c3..e2e8b8c83 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -45,7 +45,14 @@ bool VaultClient::InitPki(const std::string& domains, CACertificateConfig* external_ca, ServerCertificateConfig* external_cert) { if (!root_client_->is_authenticated()) { - CRANE_ERROR("Root client is not authenticated"); + CRANE_ERROR("Vault client is not authenticated"); + return false; + } + + if (!GetVaultHealth_()) { + CRANE_ERROR( + "Vault client connect fail, Please check if it is started, initialized " + "and unsealed."); return false; } @@ -56,13 +63,15 @@ bool VaultClient::InitPki(const std::string& domains, pki_external_ = std::make_unique( Vault::Pki{*root_client_, Vault::SecretMount{"pki_external"}}); - // 检查CA是否存在，存在则写入文件中，不存在则创建CA + // Check if the CA exists. If it exists, write it into a file; + // if not, create the CA. if (!IssureExternalCa_(domains, external_ca)) return false; - // 创建role + // Create external role if (!CreateRole_("external", domains)) return false; - // 检查external_pem，external_key是否存在，不存在则创建，写入文件中 + // Check if external_pem and external_key exist. If they do not exist, create + // them and write them into a file. if (!IssureExternalCert_("external", domains, external_cert)) return false; return true; @@ -139,8 +148,8 @@ bool VaultClient::IssureExternalCa_(const std::string& domains, } external_ca->CACertContent = external_ca_pem; - if (external_ca->CACertFilePath.empty()) { - CRANE_ERROR("ExternalCAFilePath is empty"); + if (external_ca->CACertContent.empty()) { + CRANE_ERROR("CACertContent is empty"); return false; } @@ -198,6 +207,11 @@ bool VaultClient::IsCertAllowed(const std::string& serial_number) { return true; } +std::optional VaultClient::GetVaultHealth_() { + return Vault::HttpConsumer::get(*root_client_, + GetUrl_("/v1/sys/", Vault::Path{"health"})); +} + bool VaultClient::CreateRole_(const std::string& role_name, const std::string& domains) { nlohmann::json::value_type data; @@ -250,7 +264,6 @@ bool VaultClient::IssureExternalCert_(const std::string& role_name, external_cert->ServerCertContent = external_pem; external_cert->ServerKeyContent = external_key; - if (!util::os::SaveFile(external_cert->ServerCertFilePath, external_pem, 0644)) return false; diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index 3dd8046b6..ca84367d1 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -57,6 +57,8 @@ class VaultClient { bool IsCertAllowed(const std::string& serial_number); private: + std::optional GetVaultHealth_(); + bool IssureExternalCa_(const std::string& domains, CACertificateConfig* external_ca); From 85ff0802f26185241f673c7d48815e30beaa2715 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 6 Jan 2025 11:12:25 +0800 Subject: [PATCH 44/62] fix: When revoked is empty, a 404 error is returned, causing the request to fail. --- src/Utilities/VaultClient/VaultClient.cpp | 30 +++++++++++++++++-- .../VaultClient/include/crane/VaultClient.h | 3 ++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index e2e8b8c83..d835486d9 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -281,9 +281,33 @@ bool VaultClient::IssureExternalCert_(const std::string& role_name, } std::optional VaultClient::ListRevokeCertificate_() { - return Vault::HttpConsumer::list( - *root_client_, GetPkiUrl_(Vault::SecretMount{"pki_external"}, - Vault::Path{"certs/revoked"})); + return list_(*root_client_, GetPkiUrl_(Vault::SecretMount{"pki_external"}, + Vault::Path{"certs/revoked"})); +} + +std::optional VaultClient::list_(const Vault::Client& client, + const Vault::Url& url) { + if (!client.is_authenticated()) { + return std::nullopt; + } + + auto response = client.getHttpClient().list(url, client.getToken(), + client.getNamespace()); + + if (Vault::HttpClient::is_success(response)) { + return {response.value().body.value()}; + } + + // Do not return an error when revoked is empty. + if (response) { + auto jsonResponse = nlohmann::json::parse(response.value().body.value()); + if (jsonResponse.contains("errors") && jsonResponse["errors"].is_array() && + jsonResponse["errors"].empty()) + return {response.value().body.value()}; + client.getHttpClient().handleResponseError(response.value()); + } + + return std::nullopt; } Vault::Url VaultClient::GetPkiUrl_(const Vault::SecretMount secret_mount, diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index ca84367d1..fe7450e54 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -70,6 +70,9 @@ class VaultClient { std::optional ListRevokeCertificate_(); + std::optional list_(const Vault::Client& client, + const Vault::Url& url); + Vault::Url GetUrl_(const std::string& base, const Vault::Path& path) const; Vault::Url GetPkiUrl_(const Vault::SecretMount secret_mount, From 73f4eb7c890449d1b874d9f55153dc688148cb1d Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:16:43 +0800 Subject: [PATCH 45/62] refactor config --- etc/config.yaml | 12 +++++------ src/CraneCtld/CraneCtld.cpp | 21 ++++++++++++++++--- src/CraneCtld/CtldPublicDefs.h | 1 + .../RpcService/CtldForCforedServer.cpp | 2 -- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/etc/config.yaml b/etc/config.yaml index 52d16a59a..665225fbc 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -25,13 +25,13 @@ SSL: Vault: Addr: 127.0.0.1 Port: 8200 - Token: REMOVED # Token cannot be leaked + TokenPath: /etc/crane/vault_token.txt # Token cannot be leaked DomainSuffix: crane.com - Nodes: "cn[15-18]" - - ExternalCertFilePath: /etc/crane/tls/external.pem - ExternalKeyFilePath: /etc/crane/tls/external.key - ExternalCaFilePath: /etc/crane/tls/external_ca.pem + Tls: false +# Nodes: "cn[15-18]" + ExternalCertFilePath: /etc/crane/pki/external.pem + ExternalKeyFilePath: /etc/crane/pki/external.key + ExternalCaFilePath: /etc/crane/pki/external_ca.pem # Ctld settings # the listening address of control machine diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 537d593f0..6d416424c 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -305,8 +305,23 @@ void ParseConfig(int argc, char** argv) { if (vault_config["Port"]) g_config.VaultConf.Port = vault_config["Port"].as(); - if (vault_config["Token"]) - g_config.VaultConf.Token = vault_config["Token"].as(); + if (vault_config["Tls"]) + g_config.VaultConf.Tls = vault_config["Tls"].as(); + else + g_config.VaultConf.Tls = false; + + if (vault_config["TokenPath"]) { + std::string token_path = vault_config["TokenPath"].as(); + try { + g_config.VaultConf.Token = util::ReadFileIntoString(token_path); + } catch (const std::exception& e) { + CRANE_ERROR("Read vault token file error: {}", e.what()); + std::exit(1); + } + } else { + CRANE_ERROR("vault token path is empty"); + std::exit(1); + } if (vault_config["DomainSuffix"]) g_config.VaultConf.DomainSuffix = @@ -857,7 +872,7 @@ void InitializeCtldGlobalVariables() { g_vault_client = std::make_unique( g_config.VaultConf.Token, g_config.VaultConf.Addr, - g_config.VaultConf.Port); + g_config.VaultConf.Port, g_config.VaultConf.Tls); if (!g_vault_client->InitPki(g_config.VaultConf.DomainSuffix, &g_config.VaultConf.ExternalCACerts, &g_config.VaultConf.ExternalCerts)) diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index ae90a705e..93b025549 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -113,6 +113,7 @@ struct Config { std::string Addr; std::string Port; std::string Token; + bool Tls; std::string DomainSuffix; std::unordered_set AllowedNodes; diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index 705aa98e4..56c5efb95 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -234,8 +234,6 @@ grpc::Status CtldForCforedServiceImpl::SignUserCertificate( } } - CRANE_INFO("Resolve hostname from address: {} -> {}", client_address, - hostname); std::vector name_list = absl::StrSplit(hostname, "."); if (!resolve_result || !g_config.VaultConf.AllowedNodes.contains(name_list[0])) { From 03fffeecfd9dc8aabf66af9a432ebe95fc006637 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:42:47 +0800 Subject: [PATCH 46/62] refactor vault client --- src/Utilities/VaultClient/VaultClient.cpp | 147 +----------------- .../VaultClient/include/crane/VaultClient.h | 12 +- 2 files changed, 2 insertions(+), 157 deletions(-) diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index d835486d9..fa9e09ed8 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -41,9 +41,7 @@ VaultClient::VaultClient(const std::string& root_token, config, tokenStrategy, httpErrorCallback, responseCallback}); } -bool VaultClient::InitPki(const std::string& domains, - CACertificateConfig* external_ca, - ServerCertificateConfig* external_cert) { +bool VaultClient::InitPki() { if (!root_client_->is_authenticated()) { CRANE_ERROR("Vault client is not authenticated"); return false; @@ -63,17 +61,6 @@ bool VaultClient::InitPki(const std::string& domains, pki_external_ = std::make_unique( Vault::Pki{*root_client_, Vault::SecretMount{"pki_external"}}); - // Check if the CA exists. If it exists, write it into a file; - // if not, create the CA. - if (!IssureExternalCa_(domains, external_ca)) return false; - - // Create external role - if (!CreateRole_("external", domains)) return false; - - // Check if external_pem and external_key exist. If they do not exist, create - // them and write them into a file. - if (!IssureExternalCert_("external", domains, external_cert)) return false; - return true; } @@ -100,70 +87,6 @@ std::expected VaultClient::Sign( return SignResponse{data["serial_number"], data["certificate"]}; } -bool VaultClient::IssureExternalCa_(const std::string& domains, - CACertificateConfig* external_ca) { - nlohmann::json::value_type data; - std::optional response; - - if (!external_ca->CACertContent.empty()) return true; - - try { - response = pki_external_->readCACertificate(); - if (!response) { - CRANE_ERROR("Failed to read external CA certificate"); - return false; - } - - std::string external_ca_pem = response.value(); - - if (external_ca_pem == "") { - response = pki_external_->generateIntermediate( - Vault::KeyType{"internal"}, - Vault::Parameters({{"common_name", std::format("*.{}", domains)}, - {"issuer_name", "CraneSched_internal_CA"}, - {"not_after", "9999-12-31T23:59:59Z"}})); - - data = nlohmann::json::parse(response.value())["data"]; - std::string external_ca_csr = data["csr"]; - - response = pki_root_->signIntermediate( - Vault::Parameters({{"csr", external_ca_csr}, - {"format", "pem_bundle"}, - {"not_after", "9999-12-31T23:59:59Z"}})); - if (!response) { - CRANE_ERROR("Failed to sign external CA certificate"); - return false; - } - - data = nlohmann::json::parse(response.value())["data"]; - - external_ca_pem = data["certificate"]; - - response = pki_external_->setSignedIntermediate( - Vault::Parameters({{"certificate", external_ca_pem}})); - if (!response) { - CRANE_ERROR("Failed to set signed external CA certificate"); - return false; - } - } - - external_ca->CACertContent = external_ca_pem; - if (external_ca->CACertContent.empty()) { - CRANE_ERROR("CACertContent is empty"); - return false; - } - - if (!util::os::SaveFile(external_ca->CACertFilePath, external_ca_pem, 0644)) - return false; - - } catch (const std::exception& e) { - CRANE_ERROR("Failed to issue external CA certificate: {}", e.what()); - return false; - } - - return true; -} - bool VaultClient::RevokeCert(const std::string& serial_number) { try { auto response = pki_external_->revokeCertificate( @@ -212,74 +135,6 @@ std::optional VaultClient::GetVaultHealth_() { GetUrl_("/v1/sys/", Vault::Path{"health"})); } -bool VaultClient::CreateRole_(const std::string& role_name, - const std::string& domains) { - nlohmann::json::value_type data; - std::optional response; - - try { - response = pki_external_->readRole(Vault::Path{role_name}); - if (response) return true; - - response = pki_external_->createRole( - Vault::Path{role_name}, - Vault::Parameters{{"allowed_domains", domains}, - {"allow_subdomains", "true"}, - {"allow_glob_domains", "true"}, - {"not_after", "9999-12-31T23:59:59Z"}}); - if (!response) { - CRANE_ERROR("Failed to create role {}", role_name); - return false; - } - } catch (const std::exception& e) { - CRANE_ERROR("Failed to create role {}: {}", role_name, e.what()); - return false; - } - - return true; -} - -bool VaultClient::IssureExternalCert_(const std::string& role_name, - const std::string& domains, - ServerCertificateConfig* external_cert) { - nlohmann::json::value_type data; - std::optional response; - - if (!external_cert->ServerCertContent.empty()) return true; - - try { - response = pki_external_->issue( - Vault::Path(role_name), - Vault::Parameters{{"common_name", std::format("external.{}", domains)}, - {"alt_names", std::format("localhost,*.{}", domains)}, - {"exclude_cn_from_sans", "true"}}); - if (!response) { - CRANE_ERROR("Failed to issue external certificate"); - return false; - } - - data = nlohmann::json::parse(response.value())["data"]; - std::string external_pem = data["certificate"]; - std::string external_key = data["private_key"]; - - external_cert->ServerCertContent = external_pem; - external_cert->ServerKeyContent = external_key; - if (!util::os::SaveFile(external_cert->ServerCertFilePath, external_pem, - 0644)) - return false; - - if (!util::os::SaveFile(external_cert->ServerKeyFilePath, external_key, - 0600)) - return false; - - } catch (const std::exception& e) { - CRANE_ERROR("Failed to issue external certificate: {}", e.what()); - return false; - } - - return true; -} - std::optional VaultClient::ListRevokeCertificate_() { return list_(*root_client_, GetPkiUrl_(Vault::SecretMount{"pki_external"}, Vault::Path{"certs/revoked"})); diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index fe7450e54..b850b78d5 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -45,8 +45,7 @@ class VaultClient { VaultClient(const std::string& root_token, const std::string& address, const std::string& port, bool tls = false); - bool InitPki(const std::string& domains, CACertificateConfig* external_ca, - ServerCertificateConfig* external_cert); + bool InitPki(); std::expected Sign(const std::string& csr_content, const std::string& common_name, @@ -59,15 +58,6 @@ class VaultClient { private: std::optional GetVaultHealth_(); - bool IssureExternalCa_(const std::string& domains, - CACertificateConfig* external_ca); - - bool CreateRole_(const std::string& role_name, const std::string& domains); - - bool IssureExternalCert_(const std::string& role_name, - const std::string& domains, - ServerCertificateConfig* external_cert); - std::optional ListRevokeCertificate_(); std::optional list_(const Vault::Client& client, From 18467eca916ca70bd0371bdee7d4b925bcf05544 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:50:45 +0800 Subject: [PATCH 47/62] refactor --- src/CraneCtld/CraneCtld.cpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 6d416424c..49e3f5710 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -347,8 +347,8 @@ void ParseConfig(int argc, char** argv) { external_certs.ServerCertContent = util::ReadFileIntoString(external_certs.ServerCertFilePath); } catch (const std::exception& e) { - CRANE_DEBUG("Read cert file error: {}", e.what()); - // std::exit(1); + CRANE_ERROR("Read external cert file error: {}", e.what()); + std::exit(1); } } else { CRANE_ERROR("ExternalCertFilePath is empty"); @@ -363,8 +363,8 @@ void ParseConfig(int argc, char** argv) { external_certs.ServerKeyContent = util::ReadFileIntoString(external_certs.ServerKeyFilePath); } catch (const std::exception& e) { - CRANE_DEBUG("Read cert file error: {}", e.what()); - // std::exit(1); + CRANE_ERROR("Read external key file error: {}", e.what()); + std::exit(1); } } else { CRANE_ERROR("ExternalKeyFilePath is empty"); @@ -379,8 +379,8 @@ void ParseConfig(int argc, char** argv) { external_ca_certs.CACertContent = util::ReadFileIntoString(external_ca_certs.CACertFilePath); } catch (const std::exception& e) { - CRANE_DEBUG("Read cert file error: {}", e.what()); - // std::exit(1); + CRANE_ERROR("Read external ca file error: {}", e.what()); + std::exit(1); } } else { CRANE_ERROR("ExternalCAFilePath is empty"); @@ -873,10 +873,7 @@ void InitializeCtldGlobalVariables() { g_vault_client = std::make_unique( g_config.VaultConf.Token, g_config.VaultConf.Addr, g_config.VaultConf.Port, g_config.VaultConf.Tls); - if (!g_vault_client->InitPki(g_config.VaultConf.DomainSuffix, - &g_config.VaultConf.ExternalCACerts, - &g_config.VaultConf.ExternalCerts)) - std::exit(1); + if (!g_vault_client->InitPki()) std::exit(1); // Account manager must be initialized before Task Scheduler // since the recovery stage of the task scheduler will acquire From 5dcee26d69b5bbe267d7981e21f45a9b8f532a3e Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:27:12 +0800 Subject: [PATCH 48/62] refactor domainsuffix config --- etc/config.yaml | 4 ++-- src/CraneCtld/AccountManager.cpp | 4 ++-- src/CraneCtld/CraneCtld.cpp | 7 +++---- src/CraneCtld/CtldPublicDefs.h | 3 +-- src/CraneCtld/RpcService/CranedKeeper.cpp | 4 ++-- src/Craned/CforedClient.cpp | 2 +- src/Craned/Craned.cpp | 7 +++---- src/Craned/CranedPublicDefs.h | 3 ++- src/Craned/CranedServer.cpp | 4 ++-- src/Craned/CtldClient.cpp | 11 +++++++---- 10 files changed, 25 insertions(+), 24 deletions(-) diff --git a/etc/config.yaml b/etc/config.yaml index 665225fbc..77438746c 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -20,19 +20,19 @@ SSL: CforedCertFilePath: /etc/crane/tls/cfored.pem CforedKeyFilePath: /etc/crane/tls/cfored.key InternalCaFilePath: /etc/crane/tls/internal_ca.pem - DomainSuffix: crane.com Vault: Addr: 127.0.0.1 Port: 8200 TokenPath: /etc/crane/vault_token.txt # Token cannot be leaked - DomainSuffix: crane.com Tls: false # Nodes: "cn[15-18]" ExternalCertFilePath: /etc/crane/pki/external.pem ExternalKeyFilePath: /etc/crane/pki/external.key ExternalCaFilePath: /etc/crane/pki/external_ca.pem +DomainSuffix: crane.com + # Ctld settings # the listening address of control machine CraneCtldListenAddr: 0.0.0.0 diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 22cabcd7d..249a3cdad 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1042,8 +1042,8 @@ AccountManager::CraneExpected AccountManager::SignUserCertificate( if (!op_user->serial_number.empty()) return std::unexpected(CraneErrCode::ERR_DUPLICATE_CERTIFICATE); - std::string common_name = std::format("{}.{}.{}", uid, op_user->name, - g_config.VaultConf.DomainSuffix); + std::string common_name = + std::format("{}.{}.{}", uid, op_user->name, g_config.DomainSuffix); auto sign_response = g_vault_client->Sign(csr_content, common_name, alt_names); if (!sign_response) diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 49e3f5710..722e93891 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -174,10 +174,6 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.UseTls = true; - if (ssl_config["DomainSuffix"]) - g_config.ListenConf.TlsCerts.DomainSuffix = - ssl_config["DomainSuffix"].as(); - if (ssl_config["InternalCaFilePath"]) { std::string internalCaFilePath = ssl_config["InternalCaFilePath"].as(); @@ -291,6 +287,9 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.UseTls = false; } + if (config["DomainSuffix"]) + g_config.DomainSuffix = config["DomainSuffix"].as(); + if (config["Vault"]) { const auto& vault_config = config["Vault"]; diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 93b025549..06575ba0d 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -92,7 +92,6 @@ struct Config { bool UseTls{false}; struct TlsCertsConfig { std::string InternalCaContent; - std::string DomainSuffix; ServerCertificateConfig ExternalCerts; ServerCertificateConfig InternalCerts; ClientCertificateConfig CranedClientCerts; @@ -114,7 +113,6 @@ struct Config { std::string Port; std::string Token; bool Tls; - std::string DomainSuffix; std::unordered_set AllowedNodes; ServerCertificateConfig ExternalCerts; @@ -122,6 +120,7 @@ struct Config { }; VaultConfig VaultConf; + std::string DomainSuffix; struct Priority { enum TypeEnum { Basic, MultiFactor }; diff --git a/src/CraneCtld/RpcService/CranedKeeper.cpp b/src/CraneCtld/RpcService/CranedKeeper.cpp index 13941dd7e..a40b92b87 100644 --- a/src/CraneCtld/RpcService/CranedKeeper.cpp +++ b/src/CraneCtld/RpcService/CranedKeeper.cpp @@ -729,8 +729,8 @@ void CranedKeeper::ConnectCranedNode_(CranedId const &craned_id) { craned->m_channel_ = CreateTcpTlsCustomChannelByHostname( craned_id, g_config.CranedListenConf.CranedListenPort, g_config.ListenConf.TlsCerts.InternalCerts, - g_config.ListenConf.TlsCerts.CranedClientCerts, - g_config.ListenConf.TlsCerts.DomainSuffix, channel_args); + g_config.ListenConf.TlsCerts.CranedClientCerts, g_config.DomainSuffix, + channel_args); else craned->m_channel_ = CreateTcpInsecureCustomChannel( ip_addr, g_config.CranedListenConf.CranedListenPort, channel_args); diff --git a/src/Craned/CforedClient.cpp b/src/Craned/CforedClient.cpp index bc946b71f..c96e636e0 100644 --- a/src/Craned/CforedClient.cpp +++ b/src/Craned/CforedClient.cpp @@ -49,7 +49,7 @@ void CforedClient::InitChannelAndStub(const std::string& cfored_name) { cfored_name, kCforedDefaultPort, g_config.ListenConf.TlsCerts.CranedTlsCerts, g_config.ListenConf.TlsCerts.CforedClientTlsCerts, - g_config.ListenConf.TlsCerts.DomainSuffix); + g_config.DomainSuffix); else m_cfored_channel_ = CreateTcpInsecureChannel(cfored_name, kCforedDefaultPort); diff --git a/src/Craned/Craned.cpp b/src/Craned/Craned.cpp index 201835ede..b5a5aa956 100644 --- a/src/Craned/Craned.cpp +++ b/src/Craned/Craned.cpp @@ -169,6 +169,9 @@ void ParseConfig(int argc, char** argv) { if (config["CompressedRpc"]) g_config.CompressedRpc = config["CompressedRpc"].as(); + if (config["DomainSuffix"]) + g_config.DomainSuffix = config["DomainSuffix"].as(); + if (config["UseTls"] && config["UseTls"].as()) { const auto& ssl_config = config["SSL"]; @@ -180,10 +183,6 @@ void ParseConfig(int argc, char** argv) { ClientCertificateConfig& cfoed_client_certs = g_config.ListenConf.TlsCerts.CforedClientTlsCerts; - if (ssl_config["DomainSuffix"]) - g_config.ListenConf.TlsCerts.DomainSuffix = - ssl_config["DomainSuffix"].as(); - if (ssl_config["InternalCaFilePath"]) { std::string internalCaFilePath = ssl_config["InternalCaFilePath"].as(); diff --git a/src/Craned/CranedPublicDefs.h b/src/Craned/CranedPublicDefs.h index b15a870b6..c1f4aa2fe 100644 --- a/src/Craned/CranedPublicDefs.h +++ b/src/Craned/CranedPublicDefs.h @@ -57,7 +57,6 @@ struct Config { bool UseTls{false}; struct TlsCertsConfig { std::string InternalCaContent; - std::string DomainSuffix; ServerCertificateConfig CranedTlsCerts; ClientCertificateConfig InternalClientTlsCerts; ClientCertificateConfig CforedClientTlsCerts; @@ -68,6 +67,8 @@ struct Config { std::string UnixSocketListenAddr; }; + std::string DomainSuffix; + struct PluginConfig { bool Enabled{false}; std::string PlugindSockPath; diff --git a/src/Craned/CranedServer.cpp b/src/Craned/CranedServer.cpp index 9c26eaa6a..209e998c8 100644 --- a/src/Craned/CranedServer.cpp +++ b/src/Craned/CranedServer.cpp @@ -291,7 +291,7 @@ grpc::Status CranedServiceImpl::QueryTaskIdFromPortForward( remote_hostname, crane_port, g_config.ListenConf.TlsCerts.CranedTlsCerts, g_config.ListenConf.TlsCerts.InternalClientTlsCerts, - g_config.ListenConf.TlsCerts.DomainSuffix); + g_config.DomainSuffix); } else { CRANE_ERROR("Failed to resolve remote address {}.", request->ssh_remote_address()); @@ -434,7 +434,7 @@ grpc::Status CranedServiceImpl::QueryTaskEnvVariablesForward( execution_node, g_config.ListenConf.CranedListenPort, g_config.ListenConf.TlsCerts.CranedTlsCerts, g_config.ListenConf.TlsCerts.InternalClientTlsCerts, - g_config.ListenConf.TlsCerts.DomainSuffix); + g_config.DomainSuffix); else channel_of_remote_service = CreateTcpInsecureChannel( execution_node, g_config.ListenConf.CranedListenPort); diff --git a/src/Craned/CtldClient.cpp b/src/Craned/CtldClient.cpp index 06aec4df2..491f9a8e3 100644 --- a/src/Craned/CtldClient.cpp +++ b/src/Craned/CtldClient.cpp @@ -17,6 +17,7 @@ */ #include "CtldClient.h" + #include "CranedPublicDefs.h" #include "crane/GrpcHelper.h" @@ -35,11 +36,12 @@ void CtldClient::InitChannelAndStub(const std::string& server_address) { if (g_config.CompressedRpc) channel_args.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP); - if (g_config.ListenConf.UseTls) + if (g_config.ListenConf.UseTls) m_ctld_channel_ = CreateTcpTlsCustomChannelByHostname( server_address, g_config.CraneCtldForCranedPort, - g_config.ListenConf.TlsCerts.CranedTlsCerts, g_config.ListenConf.TlsCerts.InternalClientTlsCerts, - g_config.ListenConf.TlsCerts.DomainSuffix, channel_args); + g_config.ListenConf.TlsCerts.CranedTlsCerts, + g_config.ListenConf.TlsCerts.InternalClientTlsCerts, + g_config.DomainSuffix, channel_args); else m_ctld_channel_ = CreateTcpInsecureCustomChannel( server_address, g_config.CraneCtldForCranedPort, channel_args); @@ -91,7 +93,8 @@ void CtldClient::OnCraneCtldConnected() { CRANE_ERROR("Failed to register actively."); } -void CtldClient::TaskStatusChangeAsync(TaskStatusChangeQueueElem&& task_status_change) { +void CtldClient::TaskStatusChangeAsync( + TaskStatusChangeQueueElem&& task_status_change) { absl::MutexLock lock(&m_task_status_change_mtx_); m_task_status_change_list_.emplace_back(std::move(task_status_change)); } From 00c4538b0663fd8d5e9f1f7b58b25d83d8bb0010 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 6 Jan 2025 17:03:20 +0800 Subject: [PATCH 49/62] feat: SignServer --- etc/config.yaml | 1 + protos/Crane.proto | 2 + src/CraneCtld/AccountManager.cpp | 3 +- src/CraneCtld/CraneCtld.cpp | 18 +++- src/CraneCtld/CtldPublicDefs.h | 1 + src/CraneCtld/DbClient.cpp | 1 + src/CraneCtld/RpcService/CMakeLists.txt | 2 + .../RpcService/CtldForCforedServer.cpp | 49 --------- .../RpcService/CtldForCforedServer.h | 5 - src/CraneCtld/RpcService/CtldGrpcServer.cpp | 4 + src/CraneCtld/RpcService/CtldGrpcServer.h | 3 - src/CraneCtld/RpcService/SignServer.cpp | 101 ++++++++++++++++++ src/CraneCtld/RpcService/SignServer.h | 76 +++++++++++++ .../PublicHeader/include/crane/PublicHeader.h | 1 + 14 files changed, 204 insertions(+), 63 deletions(-) create mode 100644 src/CraneCtld/RpcService/SignServer.cpp create mode 100644 src/CraneCtld/RpcService/SignServer.h diff --git a/etc/config.yaml b/etc/config.yaml index 77438746c..a6e868319 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -40,6 +40,7 @@ CraneCtldListenAddr: 0.0.0.0 CraneCtldListenPort: 10011 CraneCtldForCranedListenPort: 10013 CraneCtldForCforedListenPort: 10014 +CraneCtldForSignListenPort: 10015 # debug level of cranectld CraneCtldDebugLevel: trace # file path of cranectld log file (relative to CraneBaseDir) diff --git a/protos/Crane.proto b/protos/Crane.proto index 1a31d1797..d0d501e97 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -821,7 +821,9 @@ service CraneCtldForCraned { service CraneCtldForCfored { /* RPCs called from Cfored */ rpc CforedStream(stream StreamCforedRequest) returns(stream StreamCtldReply); +} +service SignService { // RPCS called from PKI request rpc SignUserCertificate(SignUserCertificateRequest) returns (SignUserCertificateResponse); } diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 249a3cdad..3a5e5fdc0 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1042,8 +1042,7 @@ AccountManager::CraneExpected AccountManager::SignUserCertificate( if (!op_user->serial_number.empty()) return std::unexpected(CraneErrCode::ERR_DUPLICATE_CERTIFICATE); - std::string common_name = - std::format("{}.{}.{}", uid, op_user->name, g_config.DomainSuffix); + std::string common_name = std::format("{}.{}", uid, g_config.DomainSuffix); auto sign_response = g_vault_client->Sign(csr_content, common_name, alt_names); if (!sign_response) diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 722e93891..58c565cbe 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -30,10 +30,13 @@ #include "AccountMetaContainer.h" #include "CranedKeeper.h" #include "CranedMetaContainer.h" +#include "CtldForCforedServer.h" +#include "CtldForCranedServer.h" #include "CtldGrpcServer.h" #include "CtldPublicDefs.h" #include "DbClient.h" #include "EmbeddedDbClient.h" +#include "SignServer.h" #include "TaskScheduler.h" #include "crane/Network.h" #include "crane/PluginClient.h" @@ -159,6 +162,13 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.CraneCtldForCforedListenPort = kCtldForCforedDefaultPort; + if (config["CraneCtldForSignListenPort"]) + g_config.ListenConf.CraneCtldForSignListenPort = + config["CraneCtldForSignListenPort"].as(); + else + g_config.ListenConf.CraneCtldForSignListenPort = + kCtldForSignDefaultPort; + if (config["CompressedRpc"]) g_config.CompressedRpc = config["CompressedRpc"].as(); @@ -322,10 +332,6 @@ void ParseConfig(int argc, char** argv) { std::exit(1); } - if (vault_config["DomainSuffix"]) - g_config.VaultConf.DomainSuffix = - vault_config["DomainSuffix"].as(); - if (vault_config["Nodes"]) { std::string nodes = vault_config["Nodes"].as(); std::list name_list; @@ -974,6 +980,8 @@ void InitializeCtldGlobalVariables() { g_ctld_for_cfored_server = std::make_unique(g_config.ListenConf); + + g_sign_server = std::make_unique(g_config.ListenConf); } void CreateFolders() { @@ -1009,6 +1017,8 @@ int StartServer() { g_ctld_for_cfored_server->Wait(); + g_sign_server->Wait(); + DestroyCtldGlobalVariables(); return 0; diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 06575ba0d..b64bbe997 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -88,6 +88,7 @@ struct Config { std::string CraneCtldListenPort; std::string CraneCtldForCranedListenPort; std::string CraneCtldForCforedListenPort; + std::string CraneCtldForSignListenPort; bool UseTls{false}; struct TlsCertsConfig { diff --git a/src/CraneCtld/DbClient.cpp b/src/CraneCtld/DbClient.cpp index 2ec9db8f9..5f1d36382 100644 --- a/src/CraneCtld/DbClient.cpp +++ b/src/CraneCtld/DbClient.cpp @@ -649,6 +649,7 @@ void MongodbClient::ViewToUser_(const bsoncxx::document::view& user_view, user->deleted = user_view["deleted"].get_bool(); user->uid = user_view["uid"].get_int64().value; user->name = user_view["name"].get_string().value; + user->serial_number = user_view["serial_number"].get_string().value; user->default_account = user_view["default_account"].get_string().value; user->admin_level = (Ctld::User::AdminLevel)user_view["admin_level"].get_int32().value; diff --git a/src/CraneCtld/RpcService/CMakeLists.txt b/src/CraneCtld/RpcService/CMakeLists.txt index 018ec4bb7..22e794aaa 100644 --- a/src/CraneCtld/RpcService/CMakeLists.txt +++ b/src/CraneCtld/RpcService/CMakeLists.txt @@ -8,6 +8,8 @@ add_library(RpcService CtldForCforedServer.cpp CranedKeeper.h CranedKeeper.cpp + SignServer.h + SignServer.cpp ) target_include_directories(RpcService PUBLIC diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index 56c5efb95..f1b780194 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -209,55 +209,6 @@ grpc::Status CtldForCforedServiceImpl::CforedStream( } } -grpc::Status CtldForCforedServiceImpl::SignUserCertificate( - grpc::ServerContext *context, - const crane::grpc::SignUserCertificateRequest *request, - crane::grpc::SignUserCertificateResponse *response) { - if (!g_config.VaultConf.AllowedNodes.empty()) { - std::string client_address = context->peer(); - std::vector str_list = absl::StrSplit(client_address, ":"); - std::string hostname; - bool resolve_result = false; - if (str_list[0] == "ipv4") { - ipv4_t addr; - if (!crane::StrToIpv4(str_list[1], &addr)) { - CRANE_ERROR("Failed to parse ipv4 address: {}", str_list[1]); - } else { - resolve_result = crane::ResolveHostnameFromIpv4(addr, &hostname); - } - } else { - ipv6_t addr; - if (!crane::StrToIpv6(str_list[1], &addr)) { - CRANE_ERROR("Failed to parse ipv6 address: {}", str_list[1]); - } else { - resolve_result = crane::ResolveHostnameFromIpv6(addr, &hostname); - } - } - - std::vector name_list = absl::StrSplit(hostname, "."); - if (!resolve_result || - !g_config.VaultConf.AllowedNodes.contains(name_list[0])) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; - } - } - - auto result = g_account_manager->SignUserCertificate( - request->uid(), request->csr_content(), request->alt_names()); - if (!result) { - response->set_ok(false); - response->set_reason(result.error()); - } else { - response->set_ok(true); - response->set_certificate(result.value()); - response->set_external_certificate( - g_config.VaultConf.ExternalCACerts.CACertContent); - } - - return grpc::Status::OK; -} - void CtldForCforedServer::Shutdown() { m_server_->Shutdown(std::chrono::system_clock::now() + std::chrono::seconds(1)); diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.h b/src/CraneCtld/RpcService/CtldForCforedServer.h index e18048e3a..d056accef 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.h +++ b/src/CraneCtld/RpcService/CtldForCforedServer.h @@ -178,11 +178,6 @@ class CtldForCforedServiceImpl final crane::grpc::StreamCforedRequest> *stream) override; - grpc::Status SignUserCertificate( - grpc::ServerContext *context, - const crane::grpc::SignUserCertificateRequest *request, - crane::grpc::SignUserCertificateResponse *response) override; - private: CtldForCforedServer *m_ctld_server_; }; diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index 837edd9aa..cd11a66fe 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -23,6 +23,9 @@ #include "../EmbeddedDbClient.h" #include "../TaskScheduler.h" #include "CranedKeeper.h" +#include "CtldForCforedServer.h" +#include "CtldForCranedServer.h" +#include "SignServer.h" #include "crane/GrpcHelper.h" #include "crane/String.h" #include "crane/VaultClient.h" @@ -953,6 +956,7 @@ CtldServer::CtldServer() : m_service_impl_(nullptr) { g_craned_keeper->Shutdown(); g_ctld_for_cfored_server->Shutdown(); g_ctld_for_craned_server->Shutdown(); + g_sign_server->Shutdown(); p_server->Shutdown(std::chrono::system_clock::now() + std::chrono::seconds(1)); }); diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.h b/src/CraneCtld/RpcService/CtldGrpcServer.h index b32a19268..5dc5bded4 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.h +++ b/src/CraneCtld/RpcService/CtldGrpcServer.h @@ -21,9 +21,6 @@ #include "../CtldPublicDefs.h" // Precompiled header comes first! -#include "CtldForCforedServer.h" -#include "CtldForCranedServer.h" -#include "crane/Lock.h" #include "protos/Crane.grpc.pb.h" #include "protos/Crane.pb.h" diff --git a/src/CraneCtld/RpcService/SignServer.cpp b/src/CraneCtld/RpcService/SignServer.cpp new file mode 100644 index 000000000..6d4c5d529 --- /dev/null +++ b/src/CraneCtld/RpcService/SignServer.cpp @@ -0,0 +1,101 @@ +/** + * Copyright (c) 2024 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "SignServer.h" + +#include "../AccountManager.h" + +namespace Ctld { + +grpc::Status SignServiceImpl::SignUserCertificate( + grpc::ServerContext *context, + const crane::grpc::SignUserCertificateRequest *request, + crane::grpc::SignUserCertificateResponse *response) { + if (!g_config.VaultConf.AllowedNodes.empty()) { + std::string client_address = context->peer(); + std::vector str_list = absl::StrSplit(client_address, ":"); + std::string hostname; + bool resolve_result = false; + if (str_list[0] == "ipv4") { + ipv4_t addr; + if (!crane::StrToIpv4(str_list[1], &addr)) { + CRANE_ERROR("Failed to parse ipv4 address: {}", str_list[1]); + } else { + resolve_result = crane::ResolveHostnameFromIpv4(addr, &hostname); + } + } else { + ipv6_t addr; + if (!crane::StrToIpv6(str_list[1], &addr)) { + CRANE_ERROR("Failed to parse ipv6 address: {}", str_list[1]); + } else { + resolve_result = crane::ResolveHostnameFromIpv6(addr, &hostname); + } + } + + std::vector name_list = absl::StrSplit(hostname, "."); + if (!resolve_result || + !g_config.VaultConf.AllowedNodes.contains(name_list[0])) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + return grpc::Status::OK; + } + } + + auto result = g_account_manager->SignUserCertificate( + request->uid(), request->csr_content(), request->alt_names()); + if (!result) { + response->set_ok(false); + response->set_reason(result.error()); + } else { + response->set_ok(true); + response->set_certificate(result.value()); + response->set_external_certificate( + g_config.VaultConf.ExternalCACerts.CACertContent); + } + + return grpc::Status::OK; +} + +void SignServer::Shutdown() { + m_server_->Shutdown(std::chrono::system_clock::now() + + std::chrono::seconds(1)); +} + +SignServer::SignServer(const Config::CraneCtldListenConf &listen_conf) { + m_service_impl_ = std::make_unique(this); + + grpc::ServerBuilder builder; + + if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); + + ServerBuilderAddTcpInsecureListeningPort( + &builder, listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForSignListenPort); + + builder.RegisterService(m_service_impl_.get()); + m_server_ = builder.BuildAndStart(); + if (!m_server_) { + CRANE_ERROR("Cannot start gRPC server!"); + std::exit(1); + } + CRANE_INFO("CraneCtld For Sign Server is listening on {}:{} and Tls is {}", + listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldForSignListenPort, listen_conf.UseTls); +} + +} // namespace Ctld \ No newline at end of file diff --git a/src/CraneCtld/RpcService/SignServer.h b/src/CraneCtld/RpcService/SignServer.h new file mode 100644 index 000000000..d0cadf53f --- /dev/null +++ b/src/CraneCtld/RpcService/SignServer.h @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2024 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "../CtldPublicDefs.h" +// Precompiled header comes first! + +#include "protos/Crane.grpc.pb.h" +#include "protos/Crane.pb.h" + +namespace Ctld { + +using crane::grpc::Craned; +using grpc::Channel; +using grpc::Server; + +class SignServer; + +class SignServiceImpl final : public crane::grpc::SignService::Service { + public: + explicit SignServiceImpl(SignServer *server) : m_sign_server_(server) {} + + grpc::Status SignUserCertificate( + grpc::ServerContext *context, + const crane::grpc::SignUserCertificateRequest *request, + crane::grpc::SignUserCertificateResponse *response) override; + + private: + SignServer *m_sign_server_; +}; + +/*** + * Note: There should be only ONE instance of CtldServer!!!! + */ +class SignServer { + public: + /*** + * User must make sure that this constructor is called only once! + * @param listen_address The "[Address]:[Port]" of SignServer. + */ + explicit SignServer(const Config::CraneCtldListenConf &listen_conf); + + inline void Wait() { m_server_->Wait(); } + + void Shutdown(); + + private: + std::unique_ptr m_service_impl_; + std::unique_ptr m_server_; + + inline static std::mutex s_sigint_mtx; + inline static std::condition_variable s_sigint_cv; + static void signal_handler_func(int) { s_sigint_cv.notify_one(); }; + + friend class SignServiceImpl; +}; + +} // namespace Ctld + +inline std::unique_ptr g_sign_server; \ No newline at end of file diff --git a/src/Utilities/PublicHeader/include/crane/PublicHeader.h b/src/Utilities/PublicHeader/include/crane/PublicHeader.h index cdd95342d..032ebbbda 100644 --- a/src/Utilities/PublicHeader/include/crane/PublicHeader.h +++ b/src/Utilities/PublicHeader/include/crane/PublicHeader.h @@ -69,6 +69,7 @@ inline const char* kCranedDefaultPort = "10010"; inline const char* kCforedDefaultPort = "10012"; inline const char* kCtldForCranedDefaultPort = "10013"; inline const char* kCtldForCforedDefaultPort = "10014"; +inline const char* kCtldForSignDefaultPort = "10015"; inline const char* kDefaultConfigPath = "/etc/crane/config.yaml"; inline const char* kDefaultDbConfigPath = "/etc/crane/database.yaml"; From 3bad1f74147b650fffbed9db163d32f61417173b Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 7 Jan 2025 09:52:57 +0800 Subject: [PATCH 50/62] fix: Add force to handle mismatches between the database and Vault. --- protos/Crane.proto | 1 + src/CraneCtld/AccountManager.cpp | 7 ++++--- src/CraneCtld/AccountManager.h | 2 +- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/protos/Crane.proto b/protos/Crane.proto index d0d501e97..09e80e193 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -760,6 +760,7 @@ message SignUserCertificateResponse { message ResetUserCredentialRequest { string username = 1; + bool is_force = 2; } message ResetUserCredentialReply { diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 3a5e5fdc0..d289efff3 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1067,7 +1067,7 @@ AccountManager::CraneExpected AccountManager::SignUserCertificate( } AccountManager::CraneExpected AccountManager::ResetUserCertificate( - uint32_t uid, const std::string& name) { + uint32_t uid, const std::string& name, bool force) { util::write_lock_guard user_guard(m_rw_user_mutex_); CraneExpected result{}; @@ -1085,8 +1085,9 @@ AccountManager::CraneExpected AccountManager::ResetUserCertificate( !CheckIfUserHasHigherPrivThan_(*op_user, user->admin_level)) return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); - if (!g_vault_client->RevokeCert(user->serial_number)) - return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); + if (user->serial_number != "" && + !g_vault_client->RevokeCert(user->serial_number)) + if (!force) return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); // Save the serial number in the database. mongocxx::client_session::with_transaction_cb callback = diff --git a/src/CraneCtld/AccountManager.h b/src/CraneCtld/AccountManager.h index 837a48b38..99d357170 100644 --- a/src/CraneCtld/AccountManager.h +++ b/src/CraneCtld/AccountManager.h @@ -150,7 +150,7 @@ class AccountManager { const std::string& alt_names); CraneExpected ResetUserCertificate(uint32_t uid, - const std::string& name); + const std::string& name, bool force); private: void InitDataMap_(); diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index cd11a66fe..509b9cee0 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -870,8 +870,8 @@ grpc::Status CraneCtldServiceImpl::ResetUserCredential( uint32_t uid = extract_result.value(); - auto result = - g_account_manager->ResetUserCertificate(uid, request->username()); + auto result = g_account_manager->ResetUserCertificate( + uid, request->username(), request->is_force()); if (!result) { response->set_ok(false); response->set_reason(result.error()); From 726b5d67e36f7a277a710701c1c5cac5026627a7 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 7 Jan 2025 10:18:25 +0800 Subject: [PATCH 51/62] fix --- src/CraneCtld/AccountManager.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index d289efff3..72af84e5e 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1085,8 +1085,9 @@ AccountManager::CraneExpected AccountManager::ResetUserCertificate( !CheckIfUserHasHigherPrivThan_(*op_user, user->admin_level)) return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); - if (user->serial_number != "" && - !g_vault_client->RevokeCert(user->serial_number)) + if (user->serial_number == "") return result; + + if (!g_vault_client->RevokeCert(user->serial_number)) if (!force) return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); // Save the serial number in the database. From 6e9ad7b333bc275da47cb7b9af788c0c01d7d811 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:38:55 +0800 Subject: [PATCH 52/62] feat: add reset multi op --- protos/Crane.proto | 2 +- src/CraneCtld/AccountManager.cpp | 43 ++++++++++++++------- src/CraneCtld/AccountManager.h | 4 +- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 9 ++++- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/protos/Crane.proto b/protos/Crane.proto index 09e80e193..8fec2e73f 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -759,7 +759,7 @@ message SignUserCertificateResponse { } message ResetUserCredentialRequest { - string username = 1; + repeated string user_list = 1; bool is_force = 2; } diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 72af84e5e..36650eda9 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1067,7 +1067,7 @@ AccountManager::CraneExpected AccountManager::SignUserCertificate( } AccountManager::CraneExpected AccountManager::ResetUserCertificate( - uint32_t uid, const std::string& name, bool force) { + uint32_t uid, const std::vector& user_list, bool force) { util::write_lock_guard user_guard(m_rw_user_mutex_); CraneExpected result{}; @@ -1075,26 +1075,41 @@ AccountManager::CraneExpected AccountManager::ResetUserCertificate( if (!user_result) return std::unexpected(user_result.error()); const User* op_user = user_result.value(); - const User* user = GetExistedUserInfoNoLock_(name); - if (!user) return std::unexpected(CraneErrCode::ERR_INVALID_USER); - if (!CheckIfUserHasHigherPrivThan_(*op_user, User::None)) return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); - if (op_user->name != name && - !CheckIfUserHasHigherPrivThan_(*op_user, user->admin_level)) - return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); + std::vector user_ptr_list; + if (user_list.empty()) { + for (const auto& [username, user] : m_user_map_) { + user_ptr_list.emplace_back(user.get()); + } + } else { + for (const auto& username : user_list) { + const User* user = GetExistedUserInfoNoLock_(username); + if (!user) return std::unexpected(CraneErrCode::ERR_INVALID_USER); + user_ptr_list.emplace_back(user); + } + } - if (user->serial_number == "") return result; + for (const auto& user : user_ptr_list) { + if (op_user->name != user->name && + !CheckIfUserHasHigherPrivThan_(*op_user, user->admin_level)) + return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); + } - if (!g_vault_client->RevokeCert(user->serial_number)) - if (!force) return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); + for (const auto& user : user_ptr_list) { + if (!user->serial_number.empty() && + !g_vault_client->RevokeCert(user->serial_number)) + if (!force) return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); + } // Save the serial number in the database. mongocxx::client_session::with_transaction_cb callback = [&](mongocxx::client_session* session) { - g_db_client->UpdateEntityOne(MongodbClient::EntityType::USER, "$set", - name, "serial_number", ""); + for (const auto& user : user_ptr_list) { + g_db_client->UpdateEntityOne(MongodbClient::EntityType::USER, "$set", + user->name, "serial_number", ""); + } }; // Update to database @@ -1102,7 +1117,9 @@ AccountManager::CraneExpected AccountManager::ResetUserCertificate( return std::unexpected(CraneErrCode::ERR_UPDATE_DATABASE); } - m_user_map_[name]->serial_number = ""; + for (const auto& user : user_ptr_list) { + m_user_map_[user->name]->serial_number = ""; + } return result; } diff --git a/src/CraneCtld/AccountManager.h b/src/CraneCtld/AccountManager.h index 99d357170..9a037b00e 100644 --- a/src/CraneCtld/AccountManager.h +++ b/src/CraneCtld/AccountManager.h @@ -149,8 +149,8 @@ class AccountManager { const std::string& csr_content, const std::string& alt_names); - CraneExpected ResetUserCertificate(uint32_t uid, - const std::string& name, bool force); + CraneExpected ResetUserCertificate( + uint32_t uid, const std::vector& user_list, bool force); private: void InitDataMap_(); diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index 509b9cee0..f801614cf 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -870,8 +870,13 @@ grpc::Status CraneCtldServiceImpl::ResetUserCredential( uint32_t uid = extract_result.value(); - auto result = g_account_manager->ResetUserCertificate( - uid, request->username(), request->is_force()); + std::vector user_list; + for (const auto &username : request->user_list()) { + user_list.emplace_back(username); + } + + auto result = g_account_manager->ResetUserCertificate(uid, user_list, + request->is_force()); if (!result) { response->set_ok(false); response->set_reason(result.error()); From 9d84d2a52cf1a31991ae2b8070ae8babbcaccb92 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:25:25 +0800 Subject: [PATCH 53/62] refactor --- protos/PublicDefs.proto | 27 +++++++++---------- src/CraneCtld/AccountManager.cpp | 10 ++----- src/CraneCtld/CtldPublicDefs.h | 2 -- .../PublicHeader/include/crane/GrpcHelper.h | 3 --- src/Utilities/VaultClient/VaultClient.cpp | 2 -- .../VaultClient/include/crane/VaultClient.h | 1 - 6 files changed, 14 insertions(+), 31 deletions(-) diff --git a/protos/PublicDefs.proto b/protos/PublicDefs.proto index e86d822ac..1cc0cf7cd 100644 --- a/protos/PublicDefs.proto +++ b/protos/PublicDefs.proto @@ -276,11 +276,10 @@ message TaskInfo { string craned_list = 36; } - // The time of different nodes across the whole cluster might not always be - // synchronized. If the time on the front end node is more than several - // seconds ahead of the CraneCtld node, a negative elapsed time might occur. - // To avoid this, the elapsed time of a task is calculated on the CraneCtld - // side. + // The time of different nodes across the whole cluster might not always be synchronized. + // If the time on the front end node is more than several seconds ahead of the CraneCtld node, + // a negative elapsed time might occur. + // To avoid this, the elapsed time of a task is calculated on the CraneCtld side. google.protobuf.Duration elapsed_time = 37; repeated string execution_node = 38; } @@ -439,17 +438,15 @@ message AccountInfo { bool blocked = 10; } -// Note: UserInfo DIFFERS from the `User` struct in C++ code and database -// representation -// and is ONLY used for communication between CraneCtld and cacctmgr -// command. If an user belongs to multiple accounts, There will be -// multiple `UserInfo` messages with `account` pointing to each account. +// Note: UserInfo DIFFERS from the `User` struct in C++ code and database representation +// and is ONLY used for communication between CraneCtld and cacctmgr command. +// If an user belongs to multiple accounts, There will be multiple `UserInfo` +// messages with `account` pointing to each account. // For example, if a user (uid=1) belongs to accounts `1,2,3`, -// there will be three `UserInfo` messages: (uid=1, account=1), (uid=1, -// account=2), (uid=1, account=3). The c++ code and database -// representation use a Map to contain in -// ONE UserInfo message all the information belonging to different -// accounts. +// there will be three `UserInfo` messages: (uid=1, account=1), (uid=1, account=2), +// (uid=1, account=3). +// The c++ code and database representation use a Map to contain +// in ONE UserInfo message all the information belonging to different accounts. message UserInfo { enum AdminLevel { None = 0; diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 36650eda9..6a6b841f6 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1075,8 +1075,8 @@ AccountManager::CraneExpected AccountManager::ResetUserCertificate( if (!user_result) return std::unexpected(user_result.error()); const User* op_user = user_result.value(); - if (!CheckIfUserHasHigherPrivThan_(*op_user, User::None)) - return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); + result = CheckIfUserHasHigherPrivThan_(*op_user, User::None); + if (!result) return result; std::vector user_ptr_list; if (user_list.empty()) { @@ -1091,12 +1091,6 @@ AccountManager::CraneExpected AccountManager::ResetUserCertificate( } } - for (const auto& user : user_ptr_list) { - if (op_user->name != user->name && - !CheckIfUserHasHigherPrivThan_(*op_user, user->admin_level)) - return std::unexpected(CraneErrCode::ERR_PERMISSION_USER); - } - for (const auto& user : user_ptr_list) { if (!user->serial_number.empty() && !g_vault_client->RevokeCert(user->serial_number)) diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index b64bbe997..26e710dca 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -93,14 +93,12 @@ struct Config { bool UseTls{false}; struct TlsCertsConfig { std::string InternalCaContent; - ServerCertificateConfig ExternalCerts; ServerCertificateConfig InternalCerts; ClientCertificateConfig CranedClientCerts; ClientCertificateConfig CforedClientCerts; }; TlsCertsConfig TlsCerts; - std::string JwtSecretContent; }; CraneCtldListenConf ListenConf; diff --git a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h index cb0475303..469a5e144 100644 --- a/src/Utilities/PublicHeader/include/crane/GrpcHelper.h +++ b/src/Utilities/PublicHeader/include/crane/GrpcHelper.h @@ -19,9 +19,6 @@ #pragma once #include -#include -#include -#include #include #include "crane/Network.h" diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index fa9e09ed8..4afd207e1 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -56,8 +56,6 @@ bool VaultClient::InitPki() { pki_root_ = std::make_unique( Vault::Pki{*root_client_, Vault::SecretMount{"pki"}}); - pki_internal_ = std::make_unique( - Vault::Pki{*root_client_, Vault::SecretMount{"pki_internal"}}); pki_external_ = std::make_unique( Vault::Pki{*root_client_, Vault::SecretMount{"pki_external"}}); diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index b850b78d5..e78c69da5 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -70,7 +70,6 @@ class VaultClient { std::unique_ptr root_client_; std::unique_ptr pki_root_; - std::unique_ptr pki_internal_; std::unique_ptr pki_external_; // TODO: 采用并行容器，提高性能 From 1995a12a6984cddf8acff6d73220fe1feac8f644 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:35:02 +0800 Subject: [PATCH 54/62] feat: add grpc::UNAUTHENTICATED errcode --- src/CraneCtld/RpcService/CtldGrpcServer.cpp | 47 +++++++++++---------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldGrpcServer.cpp index f801614cf..ea1437047 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldGrpcServer.cpp @@ -40,7 +40,7 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTask( if (!extract_result) { response->set_ok(false); response->set_reason("Permission Denied."); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -88,7 +88,7 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTasks( for (int i = 0; i < task_count; i++) { response->mutable_reason_list()->Add("Permission Denied."); } - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -123,7 +123,7 @@ grpc::Status CraneCtldServiceImpl::CancelTask( response->add_not_cancelled_tasks(task_id); response->add_not_cancelled_reasons("Permission Denied."); } - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -144,7 +144,8 @@ grpc::Status CraneCtldServiceImpl::QueryCranedInfo( const crane::grpc::QueryCranedInfoRequest *request, crane::grpc::QueryCranedInfoReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) return grpc::Status::OK; + if (!extract_result) + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); if (request->craned_name().empty()) { *response = g_meta_container->QueryAllCranedInfo(); @@ -160,7 +161,8 @@ grpc::Status CraneCtldServiceImpl::QueryPartitionInfo( const crane::grpc::QueryPartitionInfoRequest *request, crane::grpc::QueryPartitionInfoReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) return grpc::Status::OK; + if (!extract_result) + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); if (request->partition_name().empty()) { *response = g_meta_container->QueryAllPartitionInfo(); @@ -181,7 +183,7 @@ grpc::Status CraneCtldServiceImpl::ModifyTask( response->add_not_modified_tasks(task_id); response->add_not_modified_reasons("Permission Denied."); } - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -274,7 +276,7 @@ grpc::Status CraneCtldServiceImpl::ModifyNode( response->add_not_modified_nodes(crane_id); response->add_not_modified_reasons("Permission Denied."); } - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -345,7 +347,7 @@ grpc::Status CraneCtldServiceImpl::AddAccount( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -382,7 +384,7 @@ grpc::Status CraneCtldServiceImpl::AddUser( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -432,7 +434,7 @@ grpc::Status CraneCtldServiceImpl::AddQos( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -474,7 +476,7 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -500,7 +502,7 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -568,7 +570,7 @@ grpc::Status CraneCtldServiceImpl::ModifyQos( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -594,7 +596,7 @@ grpc::Status CraneCtldServiceImpl::QueryAccountInfo( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -656,7 +658,7 @@ grpc::Status CraneCtldServiceImpl::QueryUserInfo( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -722,7 +724,7 @@ grpc::Status CraneCtldServiceImpl::QueryQosInfo( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -759,7 +761,7 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -781,7 +783,7 @@ grpc::Status CraneCtldServiceImpl::DeleteUser( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -804,7 +806,7 @@ grpc::Status CraneCtldServiceImpl::DeleteQos( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -828,7 +830,7 @@ grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -865,7 +867,7 @@ grpc::Status CraneCtldServiceImpl::ResetUserCredential( if (!extract_result) { response->set_ok(false); response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); } uint32_t uid = extract_result.value(); @@ -892,7 +894,8 @@ grpc::Status CraneCtldServiceImpl::QueryClusterInfo( const crane::grpc::QueryClusterInfoRequest *request, crane::grpc::QueryClusterInfoReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) return grpc::Status::OK; + if (!extract_result) + return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); *response = g_meta_container->QueryClusterInfo(*request); return grpc::Status::OK; From c96dd9433fd031f0ff0461ad87221eac7877e8f1 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:14:58 +0800 Subject: [PATCH 55/62] feat: Use parallel containers to improve performance. --- .../RpcService/CtldForCforedServer.cpp | 24 ++++++++++--------- .../RpcService/CtldForCforedServer.h | 24 ++++++++++--------- .../RpcService/CtldForCranedServer.cpp | 24 ++++++++++--------- .../RpcService/CtldForCranedServer.h | 24 ++++++++++--------- src/CraneCtld/RpcService/SignServer.h | 4 ---- src/Utilities/VaultClient/CMakeLists.txt | 1 + src/Utilities/VaultClient/VaultClient.cpp | 15 +++--------- .../VaultClient/include/crane/VaultClient.h | 8 +++---- 8 files changed, 60 insertions(+), 64 deletions(-) diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index f1b780194..f8ac28c02 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -1,17 +1,19 @@ /** - * Copyright (c) 2023 Peking University and Peking University + * Copyright (c) 2024 Peking University and Peking University * Changsha Institute for Computing and Digital Economy * - * CraneSched is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of - * the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, - * WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . */ #include "CtldForCforedServer.h" diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.h b/src/CraneCtld/RpcService/CtldForCforedServer.h index d056accef..411c44eb7 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.h +++ b/src/CraneCtld/RpcService/CtldForCforedServer.h @@ -1,17 +1,19 @@ /** - * Copyright (c) 2023 Peking University and Peking University + * Copyright (c) 2024 Peking University and Peking University * Changsha Institute for Computing and Digital Economy * - * CraneSched is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of - * the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, - * WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . */ #pragma once diff --git a/src/CraneCtld/RpcService/CtldForCranedServer.cpp b/src/CraneCtld/RpcService/CtldForCranedServer.cpp index 39563c2be..3fb3e5e6a 100644 --- a/src/CraneCtld/RpcService/CtldForCranedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCranedServer.cpp @@ -1,17 +1,19 @@ /** - * Copyright (c) 2023 Peking University and Peking University + * Copyright (c) 2024 Peking University and Peking University * Changsha Institute for Computing and Digital Economy * - * CraneSched is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of - * the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, - * WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . */ #include "CtldForCranedServer.h" diff --git a/src/CraneCtld/RpcService/CtldForCranedServer.h b/src/CraneCtld/RpcService/CtldForCranedServer.h index 2966c8a3e..9c4644405 100644 --- a/src/CraneCtld/RpcService/CtldForCranedServer.h +++ b/src/CraneCtld/RpcService/CtldForCranedServer.h @@ -1,17 +1,19 @@ /** - * Copyright (c) 2023 Peking University and Peking University + * Copyright (c) 2024 Peking University and Peking University * Changsha Institute for Computing and Digital Economy * - * CraneSched is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of - * the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, - * WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . */ #pragma once diff --git a/src/CraneCtld/RpcService/SignServer.h b/src/CraneCtld/RpcService/SignServer.h index d0cadf53f..d6f4308b9 100644 --- a/src/CraneCtld/RpcService/SignServer.h +++ b/src/CraneCtld/RpcService/SignServer.h @@ -64,10 +64,6 @@ class SignServer { std::unique_ptr m_service_impl_; std::unique_ptr m_server_; - inline static std::mutex s_sigint_mtx; - inline static std::condition_variable s_sigint_cv; - static void signal_handler_func(int) { s_sigint_cv.notify_one(); }; - friend class SignServiceImpl; }; diff --git a/src/Utilities/VaultClient/CMakeLists.txt b/src/Utilities/VaultClient/CMakeLists.txt index c003bf891..e007155ed 100644 --- a/src/Utilities/VaultClient/CMakeLists.txt +++ b/src/Utilities/VaultClient/CMakeLists.txt @@ -6,5 +6,6 @@ target_link_libraries(Utility_VaultClient PUBLIC curl vault nlohmann_json + phmap Utility_PublicHeader ) diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index 4afd207e1..3872ee70e 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -94,19 +94,13 @@ bool VaultClient::RevokeCert(const std::string& serial_number) { return false; } - { - util::write_lock_guard write_lock(rw_mutex_); - allowed_certs_.erase(serial_number); - } + allowed_certs_.erase(serial_number); return true; } bool VaultClient::IsCertAllowed(const std::string& serial_number) { - { - util::read_lock_guard read_lock(rw_mutex_); - if (allowed_certs_.contains(serial_number)) return true; - } + if (allowed_certs_.contains(serial_number)) return true; try { auto response = ListRevokeCertificate_(); @@ -120,10 +114,7 @@ bool VaultClient::IsCertAllowed(const std::string& serial_number) { return false; } - { - util::write_lock_guard write_lock(rw_mutex_); - allowed_certs_.emplace(serial_number); - } + allowed_certs_.emplace(serial_number); return true; } diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index e78c69da5..ce743d35b 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -38,7 +39,7 @@ struct SignResponse { std::string certificate; }; -using AllowedCerts = std::unordered_set; +using AllowedCerts = phmap::parallel_flat_hash_set; class VaultClient { public: @@ -72,9 +73,8 @@ class VaultClient { std::unique_ptr pki_root_; std::unique_ptr pki_external_; - // TODO: 采用并行容器，提高性能 - AllowedCerts allowed_certs_ ABSL_GUARDED_BY(rw_mutex_); - util::rw_mutex rw_mutex_; + // Use parallel containers to improve performance. + AllowedCerts allowed_certs_; std::string address_; std::string port_; From 14cd67b8dd03d121392bc411e56be669d39d5f35 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Wed, 12 Feb 2025 16:10:08 +0800 Subject: [PATCH 56/62] refactor set_external_certificate --- src/CraneCtld/RpcService/SignServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CraneCtld/RpcService/SignServer.cpp b/src/CraneCtld/RpcService/SignServer.cpp index 6d4c5d529..60fc8e704 100644 --- a/src/CraneCtld/RpcService/SignServer.cpp +++ b/src/CraneCtld/RpcService/SignServer.cpp @@ -65,7 +65,7 @@ grpc::Status SignServiceImpl::SignUserCertificate( response->set_ok(true); response->set_certificate(result.value()); response->set_external_certificate( - g_config.VaultConf.ExternalCACerts.CACertContent); + g_config.VaultConf.ExternalCerts.ServerCertContent); } return grpc::Status::OK; From b2c718777860b78ed408fbef143fa7080d6cf346 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:23:17 +0800 Subject: [PATCH 57/62] feat: Distinguish between secure and public interfaces --- etc/config.yaml | 6 +- protos/Crane.proto | 23 +-- src/CraneCtld/CraneCtld.cpp | 26 +-- src/CraneCtld/CtldPublicDefs.h | 2 +- src/CraneCtld/RpcService/CMakeLists.txt | 8 +- .../RpcService/CtldForCforedServer.cpp | 2 +- src/CraneCtld/RpcService/CtldPlainServer.cpp | 186 ++++++++++++++++++ .../{SignServer.h => CtldPlainServer.h} | 40 +++- ...tldGrpcServer.cpp => CtldSecureServer.cpp} | 144 +++----------- .../{CtldGrpcServer.h => CtldSecureServer.h} | 40 ++-- src/CraneCtld/RpcService/SignServer.cpp | 101 ---------- .../PublicHeader/include/crane/PublicHeader.h | 2 +- 12 files changed, 289 insertions(+), 291 deletions(-) create mode 100644 src/CraneCtld/RpcService/CtldPlainServer.cpp rename src/CraneCtld/RpcService/{SignServer.h => CtldPlainServer.h} (55%) rename src/CraneCtld/RpcService/{CtldGrpcServer.cpp => CtldSecureServer.cpp} (86%) rename src/CraneCtld/RpcService/{CtldGrpcServer.h => CtldSecureServer.h} (82%) delete mode 100644 src/CraneCtld/RpcService/SignServer.cpp diff --git a/etc/config.yaml b/etc/config.yaml index a6e868319..a0f043ddc 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -13,8 +13,8 @@ CraneBaseDir: /var/crane/ # Tls settings UseTls: true SSL: - CranectldInternalCertFilePath: /etc/crane/tls/cranectld.pem - CranectldInternalKeyFilePath: /etc/crane/tls/cranectld.key + CranectldInternalCertFilePath: /etc/crane/tls/internal.pem + CranectldInternalKeyFilePath: /etc/crane/tls/internal.key CranedCertFilePath: /etc/crane/tls/craned.pem CranedKeyFilePath: /etc/crane/tls/craned.key CforedCertFilePath: /etc/crane/tls/cfored.pem @@ -40,7 +40,7 @@ CraneCtldListenAddr: 0.0.0.0 CraneCtldListenPort: 10011 CraneCtldForCranedListenPort: 10013 CraneCtldForCforedListenPort: 10014 -CraneCtldForSignListenPort: 10015 +CraneCtldPlainListenPort: 10015 # debug level of cranectld CraneCtldDebugLevel: trace # file path of cranectld log file (relative to CraneBaseDir) diff --git a/protos/Crane.proto b/protos/Crane.proto index 8fec2e73f..e53bf181f 100644 --- a/protos/Crane.proto +++ b/protos/Crane.proto @@ -768,10 +768,8 @@ message ResetUserCredentialReply { ErrCode reason = 2; } -// Todo: Divide service into two parts: one for Craned and one for Crun -// We need to distinguish the message sender -// and have some kind of authentication -service CraneCtld { +// The service interface performs encryption and authentication. +service CraneCtldSecure { /* RPCs called from ccancel */ rpc CancelTask(CancelTaskRequest) returns (CancelTaskReply); @@ -780,8 +778,6 @@ service CraneCtld { rpc SubmitBatchTasks(SubmitBatchTasksRequest) returns (SubmitBatchTasksReply); /* PRCs called from ccontrol */ - rpc QueryCranedInfo(QueryCranedInfoRequest) returns (QueryCranedInfoReply); - rpc QueryPartitionInfo(QueryPartitionInfoRequest) returns (QueryPartitionInfoReply); rpc ModifyTask(ModifyTaskRequest) returns (ModifyTaskReply); rpc ModifyNode(ModifyCranedStateRequest) returns (ModifyCranedStateReply); @@ -805,12 +801,22 @@ service CraneCtld { rpc BlockAccountOrUser(BlockAccountOrUserRequest) returns (BlockAccountOrUserReply); rpc ResetUserCredential(ResetUserCredentialRequest) returns (ResetUserCredentialReply); +} + +// The service is completely open. +service CraneCtldPlain { + /* PRCs called from ccontrol */ + rpc QueryCranedInfo(QueryCranedInfoRequest) returns (QueryCranedInfoReply); + rpc QueryPartitionInfo(QueryPartitionInfoRequest) returns (QueryPartitionInfoReply); /* RPCs called from cinfo */ rpc QueryClusterInfo(QueryClusterInfoRequest) returns (QueryClusterInfoReply); /* common RPCs */ rpc QueryTasksInfo(QueryTasksInfoRequest) returns (QueryTasksInfoReply); + + /* RPCS called from PKI request */ + rpc SignUserCertificate(SignUserCertificateRequest) returns (SignUserCertificateResponse); } service CraneCtldForCraned { @@ -824,11 +830,6 @@ service CraneCtldForCfored { rpc CforedStream(stream StreamCforedRequest) returns(stream StreamCtldReply); } -service SignService { - // RPCS called from PKI request - rpc SignUserCertificate(SignUserCertificateRequest) returns (SignUserCertificateResponse); -} - service Craned { /* ----------------------------------- Called from CraneCtld ---------------------------------------------------- */ diff --git a/src/CraneCtld/CraneCtld.cpp b/src/CraneCtld/CraneCtld.cpp index 58c565cbe..a7b2bc0e7 100644 --- a/src/CraneCtld/CraneCtld.cpp +++ b/src/CraneCtld/CraneCtld.cpp @@ -32,11 +32,11 @@ #include "CranedMetaContainer.h" #include "CtldForCforedServer.h" #include "CtldForCranedServer.h" -#include "CtldGrpcServer.h" +#include "CtldPlainServer.h" #include "CtldPublicDefs.h" +#include "CtldSecureServer.h" #include "DbClient.h" #include "EmbeddedDbClient.h" -#include "SignServer.h" #include "TaskScheduler.h" #include "crane/Network.h" #include "crane/PluginClient.h" @@ -162,12 +162,11 @@ void ParseConfig(int argc, char** argv) { g_config.ListenConf.CraneCtldForCforedListenPort = kCtldForCforedDefaultPort; - if (config["CraneCtldForSignListenPort"]) - g_config.ListenConf.CraneCtldForSignListenPort = - config["CraneCtldForSignListenPort"].as(); + if (config["CraneCtldPlainListenPort"]) + g_config.ListenConf.CraneCtldPlainListenPort = + config["CraneCtldPlainListenPort"].as(); else - g_config.ListenConf.CraneCtldForSignListenPort = - kCtldForSignDefaultPort; + g_config.ListenConf.CraneCtldPlainListenPort = kCtldPlainDefaultPort; if (config["CompressedRpc"]) g_config.CompressedRpc = config["CompressedRpc"].as(); @@ -973,15 +972,16 @@ void InitializeCtldGlobalVariables() { std::exit(1); } - g_ctld_server = std::make_unique(); + g_ctld_secure_server = std::make_unique(); + + g_ctld_plain_server = + std::make_unique(g_config.ListenConf); g_ctld_for_craned_server = std::make_unique(g_config.ListenConf); g_ctld_for_cfored_server = std::make_unique(g_config.ListenConf); - - g_sign_server = std::make_unique(g_config.ListenConf); } void CreateFolders() { @@ -1011,14 +1011,14 @@ int StartServer() { InitializeCtldGlobalVariables(); - g_ctld_server->Wait(); + g_ctld_secure_server->Wait(); + + g_ctld_plain_server->Wait(); g_ctld_for_craned_server->Wait(); g_ctld_for_cfored_server->Wait(); - g_sign_server->Wait(); - DestroyCtldGlobalVariables(); return 0; diff --git a/src/CraneCtld/CtldPublicDefs.h b/src/CraneCtld/CtldPublicDefs.h index 26e710dca..173ea7087 100644 --- a/src/CraneCtld/CtldPublicDefs.h +++ b/src/CraneCtld/CtldPublicDefs.h @@ -88,7 +88,7 @@ struct Config { std::string CraneCtldListenPort; std::string CraneCtldForCranedListenPort; std::string CraneCtldForCforedListenPort; - std::string CraneCtldForSignListenPort; + std::string CraneCtldPlainListenPort; bool UseTls{false}; struct TlsCertsConfig { diff --git a/src/CraneCtld/RpcService/CMakeLists.txt b/src/CraneCtld/RpcService/CMakeLists.txt index 22e794aaa..07ae2f4b4 100644 --- a/src/CraneCtld/RpcService/CMakeLists.txt +++ b/src/CraneCtld/RpcService/CMakeLists.txt @@ -1,15 +1,15 @@ # Create a library from the RpcService sources add_library(RpcService - CtldGrpcServer.h - CtldGrpcServer.cpp + CtldSecureServer.h + CtldSecureServer.cpp + CtldPlainServer.h + CtldPlainServer.cpp CtldForCranedServer.h CtldForCranedServer.cpp CtldForCforedServer.h CtldForCforedServer.cpp CranedKeeper.h CranedKeeper.cpp - SignServer.h - SignServer.cpp ) target_include_directories(RpcService PUBLIC diff --git a/src/CraneCtld/RpcService/CtldForCforedServer.cpp b/src/CraneCtld/RpcService/CtldForCforedServer.cpp index f8ac28c02..1447dd589 100644 --- a/src/CraneCtld/RpcService/CtldForCforedServer.cpp +++ b/src/CraneCtld/RpcService/CtldForCforedServer.cpp @@ -26,7 +26,7 @@ #include "../EmbeddedDbClient.h" #include "../TaskScheduler.h" #include "CranedKeeper.h" -#include "CtldGrpcServer.h" +#include "CtldSecureServer.h" #include "crane/Network.h" #include "crane/String.h" diff --git a/src/CraneCtld/RpcService/CtldPlainServer.cpp b/src/CraneCtld/RpcService/CtldPlainServer.cpp new file mode 100644 index 000000000..9d0f9732a --- /dev/null +++ b/src/CraneCtld/RpcService/CtldPlainServer.cpp @@ -0,0 +1,186 @@ +/** + * Copyright (c) 2024 Peking University and Peking University + * Changsha Institute for Computing and Digital Economy + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "CtldPlainServer.h" + +#include "../AccountManager.h" +#include "../CranedMetaContainer.h" +#include "../EmbeddedDbClient.h" +#include "../TaskScheduler.h" +#include "crane/VaultClient.h" + +namespace Ctld { + +grpc::Status CraneCtldPlainServiceImpl::QueryCranedInfo( + grpc::ServerContext *context, + const crane::grpc::QueryCranedInfoRequest *request, + crane::grpc::QueryCranedInfoReply *response) { + if (request->craned_name().empty()) { + *response = g_meta_container->QueryAllCranedInfo(); + } else { + *response = g_meta_container->QueryCranedInfo(request->craned_name()); + } + + return grpc::Status::OK; +} + +grpc::Status CraneCtldPlainServiceImpl::QueryPartitionInfo( + grpc::ServerContext *context, + const crane::grpc::QueryPartitionInfoRequest *request, + crane::grpc::QueryPartitionInfoReply *response) { + if (request->partition_name().empty()) { + *response = g_meta_container->QueryAllPartitionInfo(); + } else { + *response = g_meta_container->QueryPartitionInfo(request->partition_name()); + } + + return grpc::Status::OK; +} + +grpc::Status CraneCtldPlainServiceImpl::QueryTasksInfo( + grpc::ServerContext *context, + const crane::grpc::QueryTasksInfoRequest *request, + crane::grpc::QueryTasksInfoReply *response) { + // Query tasks in RAM + g_task_scheduler->QueryTasksInRam(request, response); + + size_t num_limit = request->num_limit() == 0 ? kDefaultQueryTaskNumLimit + : request->num_limit(); + if (!request->filter_task_ids().empty()) + num_limit = std::min((size_t)request->filter_task_ids_size(), num_limit); + + auto *task_list = response->mutable_task_info_list(); + + auto sort_and_truncate = [](auto *task_list, size_t limit) -> void { + std::sort( + task_list->begin(), task_list->end(), + [](const crane::grpc::TaskInfo &a, const crane::grpc::TaskInfo &b) { + return (a.status() == b.status()) ? (a.priority() > b.priority()) + : (a.status() < b.status()); + }); + + if (task_list->size() > limit) + task_list->DeleteSubrange(limit, task_list->size()); + }; + + if (task_list->size() >= num_limit || + !request->option_include_completed_tasks()) { + sort_and_truncate(task_list, num_limit); + response->set_ok(true); + return grpc::Status::OK; + } + + // Query completed tasks in Mongodb + // (only for cacct, which sets `option_include_completed_tasks` to true) + if (!g_db_client->FetchJobRecords(request, response, + num_limit - task_list->size())) { + CRANE_ERROR("Failed to call g_db_client->FetchJobRecords"); + return grpc::Status::OK; + } + + sort_and_truncate(task_list, num_limit); + response->set_ok(true); + return grpc::Status::OK; +} + +grpc::Status CraneCtldPlainServiceImpl::QueryClusterInfo( + grpc::ServerContext *context, + const crane::grpc::QueryClusterInfoRequest *request, + crane::grpc::QueryClusterInfoReply *response) { + *response = g_meta_container->QueryClusterInfo(*request); + return grpc::Status::OK; +} + +grpc::Status CraneCtldPlainServiceImpl::SignUserCertificate( + grpc::ServerContext *context, + const crane::grpc::SignUserCertificateRequest *request, + crane::grpc::SignUserCertificateResponse *response) { + if (!g_config.VaultConf.AllowedNodes.empty()) { + std::string client_address = context->peer(); + std::vector str_list = absl::StrSplit(client_address, ":"); + std::string hostname; + bool resolve_result = false; + if (str_list[0] == "ipv4") { + ipv4_t addr; + if (!crane::StrToIpv4(str_list[1], &addr)) { + CRANE_ERROR("Failed to parse ipv4 address: {}", str_list[1]); + } else { + resolve_result = crane::ResolveHostnameFromIpv4(addr, &hostname); + } + } else { + ipv6_t addr; + if (!crane::StrToIpv6(str_list[1], &addr)) { + CRANE_ERROR("Failed to parse ipv6 address: {}", str_list[1]); + } else { + resolve_result = crane::ResolveHostnameFromIpv6(addr, &hostname); + } + } + + std::vector name_list = absl::StrSplit(hostname, "."); + if (!resolve_result || + !g_config.VaultConf.AllowedNodes.contains(name_list[0])) { + response->set_ok(false); + response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + return grpc::Status::OK; + } + } + + auto result = g_account_manager->SignUserCertificate( + request->uid(), request->csr_content(), request->alt_names()); + if (!result) { + response->set_ok(false); + response->set_reason(result.error()); + } else { + response->set_ok(true); + response->set_certificate(result.value()); + response->set_external_certificate( + g_config.VaultConf.ExternalCerts.ServerCertContent); + } + + return grpc::Status::OK; +} + +void CtldPlainServer::Shutdown() { + m_server_->Shutdown(std::chrono::system_clock::now() + + std::chrono::seconds(1)); +} + +CtldPlainServer::CtldPlainServer( + const Config::CraneCtldListenConf &listen_conf) { + m_service_impl_ = std::make_unique(this); + + grpc::ServerBuilder builder; + + if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); + + ServerBuilderAddTcpInsecureListeningPort( + &builder, listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldPlainListenPort); + + builder.RegisterService(m_service_impl_.get()); + m_server_ = builder.BuildAndStart(); + if (!m_server_) { + CRANE_ERROR("Cannot start gRPC server!"); + std::exit(1); + } + CRANE_INFO("CraneCtld For Ctld Plain Server is listening on {}:{}", + listen_conf.CraneCtldListenAddr, + listen_conf.CraneCtldPlainListenPort); +} + +} // namespace Ctld \ No newline at end of file diff --git a/src/CraneCtld/RpcService/SignServer.h b/src/CraneCtld/RpcService/CtldPlainServer.h similarity index 55% rename from src/CraneCtld/RpcService/SignServer.h rename to src/CraneCtld/RpcService/CtldPlainServer.h index d6f4308b9..2e2c15c11 100644 --- a/src/CraneCtld/RpcService/SignServer.h +++ b/src/CraneCtld/RpcService/CtldPlainServer.h @@ -30,11 +30,33 @@ using crane::grpc::Craned; using grpc::Channel; using grpc::Server; -class SignServer; +class CtldPlainServer; -class SignServiceImpl final : public crane::grpc::SignService::Service { +class CraneCtldPlainServiceImpl final + : public crane::grpc::CraneCtldPlain::Service { public: - explicit SignServiceImpl(SignServer *server) : m_sign_server_(server) {} + explicit CraneCtldPlainServiceImpl(CtldPlainServer *server) + : m_ctld_plain_server_(server) {} + + grpc::Status QueryTasksInfo( + grpc::ServerContext *context, + const crane::grpc::QueryTasksInfoRequest *request, + crane::grpc::QueryTasksInfoReply *response) override; + + grpc::Status QueryCranedInfo( + grpc::ServerContext *context, + const crane::grpc::QueryCranedInfoRequest *request, + crane::grpc::QueryCranedInfoReply *response) override; + + grpc::Status QueryPartitionInfo( + grpc::ServerContext *context, + const crane::grpc::QueryPartitionInfoRequest *request, + crane::grpc::QueryPartitionInfoReply *response) override; + + grpc::Status QueryClusterInfo( + grpc::ServerContext *context, + const crane::grpc::QueryClusterInfoRequest *request, + crane::grpc::QueryClusterInfoReply *response) override; grpc::Status SignUserCertificate( grpc::ServerContext *context, @@ -42,31 +64,31 @@ class SignServiceImpl final : public crane::grpc::SignService::Service { crane::grpc::SignUserCertificateResponse *response) override; private: - SignServer *m_sign_server_; + CtldPlainServer *m_ctld_plain_server_; }; /*** * Note: There should be only ONE instance of CtldServer!!!! */ -class SignServer { +class CtldPlainServer { public: /*** * User must make sure that this constructor is called only once! * @param listen_address The "[Address]:[Port]" of SignServer. */ - explicit SignServer(const Config::CraneCtldListenConf &listen_conf); + explicit CtldPlainServer(const Config::CraneCtldListenConf &listen_conf); inline void Wait() { m_server_->Wait(); } void Shutdown(); private: - std::unique_ptr m_service_impl_; + std::unique_ptr m_service_impl_; std::unique_ptr m_server_; - friend class SignServiceImpl; + friend class CraneCtldPlainServiceImpl; }; } // namespace Ctld -inline std::unique_ptr g_sign_server; \ No newline at end of file +inline std::unique_ptr g_ctld_plain_server; \ No newline at end of file diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.cpp b/src/CraneCtld/RpcService/CtldSecureServer.cpp similarity index 86% rename from src/CraneCtld/RpcService/CtldGrpcServer.cpp rename to src/CraneCtld/RpcService/CtldSecureServer.cpp index ea1437047..dcee39373 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.cpp +++ b/src/CraneCtld/RpcService/CtldSecureServer.cpp @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -#include "CtldGrpcServer.h" +#include "CtldSecureServer.h" #include "../AccountManager.h" #include "../CranedMetaContainer.h" @@ -25,14 +25,14 @@ #include "CranedKeeper.h" #include "CtldForCforedServer.h" #include "CtldForCranedServer.h" -#include "SignServer.h" +#include "CtldPlainServer.h" #include "crane/GrpcHelper.h" #include "crane/String.h" #include "crane/VaultClient.h" namespace Ctld { -grpc::Status CraneCtldServiceImpl::SubmitBatchTask( +grpc::Status CraneCtldSecureServiceImpl::SubmitBatchTask( grpc::ServerContext *context, const crane::grpc::SubmitBatchTaskRequest *request, crane::grpc::SubmitBatchTaskReply *response) { @@ -73,7 +73,7 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTask( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::SubmitBatchTasks( +grpc::Status CraneCtldSecureServiceImpl::SubmitBatchTasks( grpc::ServerContext *context, const crane::grpc::SubmitBatchTasksRequest *request, crane::grpc::SubmitBatchTasksReply *response) { @@ -114,7 +114,7 @@ grpc::Status CraneCtldServiceImpl::SubmitBatchTasks( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::CancelTask( +grpc::Status CraneCtldSecureServiceImpl::CancelTask( grpc::ServerContext *context, const crane::grpc::CancelTaskRequest *request, crane::grpc::CancelTaskReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); @@ -139,41 +139,7 @@ grpc::Status CraneCtldServiceImpl::CancelTask( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::QueryCranedInfo( - grpc::ServerContext *context, - const crane::grpc::QueryCranedInfoRequest *request, - crane::grpc::QueryCranedInfoReply *response) { - auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) - return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - - if (request->craned_name().empty()) { - *response = g_meta_container->QueryAllCranedInfo(); - } else { - *response = g_meta_container->QueryCranedInfo(request->craned_name()); - } - - return grpc::Status::OK; -} - -grpc::Status CraneCtldServiceImpl::QueryPartitionInfo( - grpc::ServerContext *context, - const crane::grpc::QueryPartitionInfoRequest *request, - crane::grpc::QueryPartitionInfoReply *response) { - auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) - return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - - if (request->partition_name().empty()) { - *response = g_meta_container->QueryAllPartitionInfo(); - } else { - *response = g_meta_container->QueryPartitionInfo(request->partition_name()); - } - - return grpc::Status::OK; -} - -grpc::Status CraneCtldServiceImpl::ModifyTask( +grpc::Status CraneCtldSecureServiceImpl::ModifyTask( grpc::ServerContext *context, const crane::grpc::ModifyTaskRequest *request, crane::grpc::ModifyTaskReply *response) { using ModifyTaskRequest = crane::grpc::ModifyTaskRequest; @@ -266,7 +232,7 @@ grpc::Status CraneCtldServiceImpl::ModifyTask( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::ModifyNode( +grpc::Status CraneCtldSecureServiceImpl::ModifyNode( grpc::ServerContext *context, const crane::grpc::ModifyCranedStateRequest *request, crane::grpc::ModifyCranedStateReply *response) { @@ -294,53 +260,7 @@ grpc::Status CraneCtldServiceImpl::ModifyNode( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::QueryTasksInfo( - grpc::ServerContext *context, - const crane::grpc::QueryTasksInfoRequest *request, - crane::grpc::QueryTasksInfoReply *response) { - // Query tasks in RAM - g_task_scheduler->QueryTasksInRam(request, response); - - size_t num_limit = request->num_limit() == 0 ? kDefaultQueryTaskNumLimit - : request->num_limit(); - if (!request->filter_task_ids().empty()) - num_limit = std::min((size_t)request->filter_task_ids_size(), num_limit); - - auto *task_list = response->mutable_task_info_list(); - - auto sort_and_truncate = [](auto *task_list, size_t limit) -> void { - std::sort( - task_list->begin(), task_list->end(), - [](const crane::grpc::TaskInfo &a, const crane::grpc::TaskInfo &b) { - return (a.status() == b.status()) ? (a.priority() > b.priority()) - : (a.status() < b.status()); - }); - - if (task_list->size() > limit) - task_list->DeleteSubrange(limit, task_list->size()); - }; - - if (task_list->size() >= num_limit || - !request->option_include_completed_tasks()) { - sort_and_truncate(task_list, num_limit); - response->set_ok(true); - return grpc::Status::OK; - } - - // Query completed tasks in Mongodb - // (only for cacct, which sets `option_include_completed_tasks` to true) - if (!g_db_client->FetchJobRecords(request, response, - num_limit - task_list->size())) { - CRANE_ERROR("Failed to call g_db_client->FetchJobRecords"); - return grpc::Status::OK; - } - - sort_and_truncate(task_list, num_limit); - response->set_ok(true); - return grpc::Status::OK; -} - -grpc::Status CraneCtldServiceImpl::AddAccount( +grpc::Status CraneCtldSecureServiceImpl::AddAccount( grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); @@ -377,7 +297,7 @@ grpc::Status CraneCtldServiceImpl::AddAccount( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::AddUser( +grpc::Status CraneCtldSecureServiceImpl::AddUser( grpc::ServerContext *context, const crane::grpc::AddUserRequest *request, crane::grpc::AddUserReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); @@ -427,7 +347,7 @@ grpc::Status CraneCtldServiceImpl::AddUser( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::AddQos( +grpc::Status CraneCtldSecureServiceImpl::AddQos( grpc::ServerContext *context, const crane::grpc::AddQosRequest *request, crane::grpc::AddQosReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); @@ -468,7 +388,7 @@ grpc::Status CraneCtldServiceImpl::AddQos( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::ModifyAccount( +grpc::Status CraneCtldSecureServiceImpl::ModifyAccount( grpc::ServerContext *context, const crane::grpc::ModifyAccountRequest *request, crane::grpc::ModifyAccountReply *response) { @@ -495,7 +415,7 @@ grpc::Status CraneCtldServiceImpl::ModifyAccount( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::ModifyUser( +grpc::Status CraneCtldSecureServiceImpl::ModifyUser( grpc::ServerContext *context, const crane::grpc::ModifyUserRequest *request, crane::grpc::ModifyUserReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); @@ -563,7 +483,7 @@ grpc::Status CraneCtldServiceImpl::ModifyUser( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::ModifyQos( +grpc::Status CraneCtldSecureServiceImpl::ModifyQos( grpc::ServerContext *context, const crane::grpc::ModifyQosRequest *request, crane::grpc::ModifyQosReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); @@ -588,7 +508,7 @@ grpc::Status CraneCtldServiceImpl::ModifyQos( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::QueryAccountInfo( +grpc::Status CraneCtldSecureServiceImpl::QueryAccountInfo( grpc::ServerContext *context, const crane::grpc::QueryAccountInfoRequest *request, crane::grpc::QueryAccountInfoReply *response) { @@ -650,7 +570,7 @@ grpc::Status CraneCtldServiceImpl::QueryAccountInfo( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::QueryUserInfo( +grpc::Status CraneCtldSecureServiceImpl::QueryUserInfo( grpc::ServerContext *context, const crane::grpc::QueryUserInfoRequest *request, crane::grpc::QueryUserInfoReply *response) { @@ -714,7 +634,7 @@ grpc::Status CraneCtldServiceImpl::QueryUserInfo( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::QueryQosInfo( +grpc::Status CraneCtldSecureServiceImpl::QueryQosInfo( grpc::ServerContext *context, const crane::grpc::QueryQosInfoRequest *request, crane::grpc::QueryQosInfoReply *response) { @@ -753,7 +673,7 @@ grpc::Status CraneCtldServiceImpl::QueryQosInfo( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::DeleteAccount( +grpc::Status CraneCtldSecureServiceImpl::DeleteAccount( grpc::ServerContext *context, const crane::grpc::DeleteAccountRequest *request, crane::grpc::DeleteAccountReply *response) { @@ -776,7 +696,7 @@ grpc::Status CraneCtldServiceImpl::DeleteAccount( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::DeleteUser( +grpc::Status CraneCtldSecureServiceImpl::DeleteUser( grpc::ServerContext *context, const crane::grpc::DeleteUserRequest *request, crane::grpc::DeleteUserReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); @@ -799,7 +719,7 @@ grpc::Status CraneCtldServiceImpl::DeleteUser( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::DeleteQos( +grpc::Status CraneCtldSecureServiceImpl::DeleteQos( grpc::ServerContext *context, const crane::grpc::DeleteQosRequest *request, crane::grpc::DeleteQosReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); @@ -822,7 +742,7 @@ grpc::Status CraneCtldServiceImpl::DeleteQos( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( +grpc::Status CraneCtldSecureServiceImpl::BlockAccountOrUser( grpc::ServerContext *context, const crane::grpc::BlockAccountOrUserRequest *request, crane::grpc::BlockAccountOrUserReply *response) { @@ -859,7 +779,7 @@ grpc::Status CraneCtldServiceImpl::BlockAccountOrUser( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::ResetUserCredential( +grpc::Status CraneCtldSecureServiceImpl::ResetUserCredential( grpc::ServerContext *context, const crane::grpc::ResetUserCredentialRequest *request, crane::grpc::ResetUserCredentialReply *response) { @@ -889,20 +809,8 @@ grpc::Status CraneCtldServiceImpl::ResetUserCredential( return grpc::Status::OK; } -grpc::Status CraneCtldServiceImpl::QueryClusterInfo( - grpc::ServerContext *context, - const crane::grpc::QueryClusterInfoRequest *request, - crane::grpc::QueryClusterInfoReply *response) { - auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) - return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - - *response = g_meta_container->QueryClusterInfo(*request); - return grpc::Status::OK; -} - std::expected -CraneCtldServiceImpl::CheckCertAllowedAndExtractUIDFromCert_( +CraneCtldSecureServiceImpl::CheckCertAllowedAndExtractUIDFromCert_( const grpc::ServerContext *context) { auto cert = context->auth_context()->FindPropertyValues("x509_pem_cert"); if (cert.empty()) return std::unexpected(false); @@ -922,8 +830,8 @@ CraneCtldServiceImpl::CheckCertAllowedAndExtractUIDFromCert_( return static_cast(std::stoul(cn_parts[0])); } -CtldServer::CtldServer() : m_service_impl_(nullptr) { - m_service_impl_ = std::make_unique(this); +CtldSecureServer::CtldSecureServer() : m_service_impl_(nullptr) { + m_service_impl_ = std::make_unique(this); grpc::ServerBuilder builder; const auto &listen_conf = g_config.ListenConf; @@ -964,13 +872,13 @@ CtldServer::CtldServer() : m_service_impl_(nullptr) { g_craned_keeper->Shutdown(); g_ctld_for_cfored_server->Shutdown(); g_ctld_for_craned_server->Shutdown(); - g_sign_server->Shutdown(); + g_ctld_plain_server->Shutdown(); p_server->Shutdown(std::chrono::system_clock::now() + std::chrono::seconds(1)); }); sigint_waiting_thread.detach(); - signal(SIGINT, &CtldServer::signal_handler_func); + signal(SIGINT, &CtldSecureServer::signal_handler_func); } } // namespace Ctld diff --git a/src/CraneCtld/RpcService/CtldGrpcServer.h b/src/CraneCtld/RpcService/CtldSecureServer.h similarity index 82% rename from src/CraneCtld/RpcService/CtldGrpcServer.h rename to src/CraneCtld/RpcService/CtldSecureServer.h index 5dc5bded4..fa2ab0c70 100644 --- a/src/CraneCtld/RpcService/CtldGrpcServer.h +++ b/src/CraneCtld/RpcService/CtldSecureServer.h @@ -30,11 +30,13 @@ using crane::grpc::Craned; using grpc::Channel; using grpc::Server; -class CtldServer; +class CtldSecureServer; -class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { +class CraneCtldSecureServiceImpl final + : public crane::grpc::CraneCtldSecure::Service { public: - explicit CraneCtldServiceImpl(CtldServer *server) : m_ctld_server_(server) {} + explicit CraneCtldSecureServiceImpl(CtldSecureServer *server) + : m_ctld_secure_server_(server) {} grpc::Status SubmitBatchTask( grpc::ServerContext *context, @@ -51,21 +53,6 @@ class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { const crane::grpc::CancelTaskRequest *request, crane::grpc::CancelTaskReply *response) override; - grpc::Status QueryTasksInfo( - grpc::ServerContext *context, - const crane::grpc::QueryTasksInfoRequest *request, - crane::grpc::QueryTasksInfoReply *response) override; - - grpc::Status QueryCranedInfo( - grpc::ServerContext *context, - const crane::grpc::QueryCranedInfoRequest *request, - crane::grpc::QueryCranedInfoReply *response) override; - - grpc::Status QueryPartitionInfo( - grpc::ServerContext *context, - const crane::grpc::QueryPartitionInfoRequest *request, - crane::grpc::QueryPartitionInfoReply *response) override; - grpc::Status ModifyTask(grpc::ServerContext *context, const crane::grpc::ModifyTaskRequest *request, crane::grpc::ModifyTaskReply *response) override; @@ -137,42 +124,37 @@ class CraneCtldServiceImpl final : public crane::grpc::CraneCtld::Service { const crane::grpc::ResetUserCredentialRequest *request, crane::grpc::ResetUserCredentialReply *response) override; - grpc::Status QueryClusterInfo( - grpc::ServerContext *context, - const crane::grpc::QueryClusterInfoRequest *request, - crane::grpc::QueryClusterInfoReply *response) override; - private: std::expected CheckCertAllowedAndExtractUIDFromCert_( const grpc::ServerContext *context); - CtldServer *m_ctld_server_; + CtldSecureServer *m_ctld_secure_server_; }; /*** * Note: There should be only ONE instance of CtldServer!!!! */ -class CtldServer { +class CtldSecureServer { public: /*** * User must make sure that this constructor is called only once! * @param listen_address The "[Address]:[Port]" of CraneCtld. */ - explicit CtldServer(); + explicit CtldSecureServer(); inline void Wait() { m_server_->Wait(); } private: - std::unique_ptr m_service_impl_; + std::unique_ptr m_service_impl_; std::unique_ptr m_server_; inline static std::mutex s_sigint_mtx; inline static std::condition_variable s_sigint_cv; static void signal_handler_func(int) { s_sigint_cv.notify_one(); }; - friend class CraneCtldServiceImpl; + friend class CraneCtldSecureServiceImpl; }; } // namespace Ctld -inline std::unique_ptr g_ctld_server; \ No newline at end of file +inline std::unique_ptr g_ctld_secure_server; \ No newline at end of file diff --git a/src/CraneCtld/RpcService/SignServer.cpp b/src/CraneCtld/RpcService/SignServer.cpp deleted file mode 100644 index 60fc8e704..000000000 --- a/src/CraneCtld/RpcService/SignServer.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright (c) 2024 Peking University and Peking University - * Changsha Institute for Computing and Digital Economy - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "SignServer.h" - -#include "../AccountManager.h" - -namespace Ctld { - -grpc::Status SignServiceImpl::SignUserCertificate( - grpc::ServerContext *context, - const crane::grpc::SignUserCertificateRequest *request, - crane::grpc::SignUserCertificateResponse *response) { - if (!g_config.VaultConf.AllowedNodes.empty()) { - std::string client_address = context->peer(); - std::vector str_list = absl::StrSplit(client_address, ":"); - std::string hostname; - bool resolve_result = false; - if (str_list[0] == "ipv4") { - ipv4_t addr; - if (!crane::StrToIpv4(str_list[1], &addr)) { - CRANE_ERROR("Failed to parse ipv4 address: {}", str_list[1]); - } else { - resolve_result = crane::ResolveHostnameFromIpv4(addr, &hostname); - } - } else { - ipv6_t addr; - if (!crane::StrToIpv6(str_list[1], &addr)) { - CRANE_ERROR("Failed to parse ipv6 address: {}", str_list[1]); - } else { - resolve_result = crane::ResolveHostnameFromIpv6(addr, &hostname); - } - } - - std::vector name_list = absl::StrSplit(hostname, "."); - if (!resolve_result || - !g_config.VaultConf.AllowedNodes.contains(name_list[0])) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); - return grpc::Status::OK; - } - } - - auto result = g_account_manager->SignUserCertificate( - request->uid(), request->csr_content(), request->alt_names()); - if (!result) { - response->set_ok(false); - response->set_reason(result.error()); - } else { - response->set_ok(true); - response->set_certificate(result.value()); - response->set_external_certificate( - g_config.VaultConf.ExternalCerts.ServerCertContent); - } - - return grpc::Status::OK; -} - -void SignServer::Shutdown() { - m_server_->Shutdown(std::chrono::system_clock::now() + - std::chrono::seconds(1)); -} - -SignServer::SignServer(const Config::CraneCtldListenConf &listen_conf) { - m_service_impl_ = std::make_unique(this); - - grpc::ServerBuilder builder; - - if (g_config.CompressedRpc) ServerBuilderSetCompression(&builder); - - ServerBuilderAddTcpInsecureListeningPort( - &builder, listen_conf.CraneCtldListenAddr, - listen_conf.CraneCtldForSignListenPort); - - builder.RegisterService(m_service_impl_.get()); - m_server_ = builder.BuildAndStart(); - if (!m_server_) { - CRANE_ERROR("Cannot start gRPC server!"); - std::exit(1); - } - CRANE_INFO("CraneCtld For Sign Server is listening on {}:{} and Tls is {}", - listen_conf.CraneCtldListenAddr, - listen_conf.CraneCtldForSignListenPort, listen_conf.UseTls); -} - -} // namespace Ctld \ No newline at end of file diff --git a/src/Utilities/PublicHeader/include/crane/PublicHeader.h b/src/Utilities/PublicHeader/include/crane/PublicHeader.h index 032ebbbda..70c4292da 100644 --- a/src/Utilities/PublicHeader/include/crane/PublicHeader.h +++ b/src/Utilities/PublicHeader/include/crane/PublicHeader.h @@ -69,7 +69,7 @@ inline const char* kCranedDefaultPort = "10010"; inline const char* kCforedDefaultPort = "10012"; inline const char* kCtldForCranedDefaultPort = "10013"; inline const char* kCtldForCforedDefaultPort = "10014"; -inline const char* kCtldForSignDefaultPort = "10015"; +inline const char* kCtldPlainDefaultPort = "10015"; inline const char* kDefaultConfigPath = "/etc/crane/config.yaml"; inline const char* kDefaultDbConfigPath = "/etc/crane/database.yaml"; From 90da9c174f949b42285f69a8306d058076f93873 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:44:25 +0800 Subject: [PATCH 58/62] fix: delete user, the cert not revoke --- src/CraneCtld/AccountManager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 6a6b841f6..fb7001549 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1857,6 +1857,9 @@ AccountManager::CraneExpected AccountManager::DeleteUser_( } } + if (res_user.deleted && !g_vault_client->RevokeCert(res_user.serial_number)) + return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); + mongocxx::client_session::with_transaction_cb callback = [&](mongocxx::client_session* session) { // delete form the parent accounts' users list From c3689dc323fe76dadbb80c0d1cdf2b18124bdc4f Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 17 Feb 2025 14:23:42 +0800 Subject: [PATCH 59/62] fix: Revoking a non-existent certificate does not throw an error. --- src/CraneCtld/RpcService/CtldSecureServer.cpp | 97 ++++--------------- src/Utilities/VaultClient/VaultClient.cpp | 67 ++++++++++++- .../VaultClient/include/crane/VaultClient.h | 9 ++ 3 files changed, 92 insertions(+), 81 deletions(-) diff --git a/src/CraneCtld/RpcService/CtldSecureServer.cpp b/src/CraneCtld/RpcService/CtldSecureServer.cpp index dcee39373..18f7f81aa 100644 --- a/src/CraneCtld/RpcService/CtldSecureServer.cpp +++ b/src/CraneCtld/RpcService/CtldSecureServer.cpp @@ -37,11 +37,8 @@ grpc::Status CraneCtldSecureServiceImpl::SubmitBatchTask( const crane::grpc::SubmitBatchTaskRequest *request, crane::grpc::SubmitBatchTaskReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason("Permission Denied."); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); if (uid != request->task().uid()) { @@ -84,12 +81,8 @@ grpc::Status CraneCtldSecureServiceImpl::SubmitBatchTasks( results.reserve(task_count); auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - for (int i = 0; i < task_count; i++) { - response->mutable_reason_list()->Add("Permission Denied."); - } + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -118,13 +111,8 @@ grpc::Status CraneCtldSecureServiceImpl::CancelTask( grpc::ServerContext *context, const crane::grpc::CancelTaskRequest *request, crane::grpc::CancelTaskReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - for (auto task_id : request->filter_task_ids()) { - response->add_not_cancelled_tasks(task_id); - response->add_not_cancelled_reasons("Permission Denied."); - } + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); if (uid != request->operator_uid()) { @@ -144,13 +132,8 @@ grpc::Status CraneCtldSecureServiceImpl::ModifyTask( crane::grpc::ModifyTaskReply *response) { using ModifyTaskRequest = crane::grpc::ModifyTaskRequest; auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - for (auto task_id : request->task_ids()) { - response->add_not_modified_tasks(task_id); - response->add_not_modified_reasons("Permission Denied."); - } + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -237,13 +220,8 @@ grpc::Status CraneCtldSecureServiceImpl::ModifyNode( const crane::grpc::ModifyCranedStateRequest *request, crane::grpc::ModifyCranedStateReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - for (auto crane_id : request->craned_ids()) { - response->add_not_modified_nodes(crane_id); - response->add_not_modified_reasons("Permission Denied."); - } + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -264,11 +242,8 @@ grpc::Status CraneCtldSecureServiceImpl::AddAccount( grpc::ServerContext *context, const crane::grpc::AddAccountRequest *request, crane::grpc::AddAccountReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -301,11 +276,8 @@ grpc::Status CraneCtldSecureServiceImpl::AddUser( grpc::ServerContext *context, const crane::grpc::AddUserRequest *request, crane::grpc::AddUserReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -351,11 +323,8 @@ grpc::Status CraneCtldSecureServiceImpl::AddQos( grpc::ServerContext *context, const crane::grpc::AddQosRequest *request, crane::grpc::AddQosReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -393,11 +362,8 @@ grpc::Status CraneCtldSecureServiceImpl::ModifyAccount( const crane::grpc::ModifyAccountRequest *request, crane::grpc::ModifyAccountReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -419,11 +385,8 @@ grpc::Status CraneCtldSecureServiceImpl::ModifyUser( grpc::ServerContext *context, const crane::grpc::ModifyUserRequest *request, crane::grpc::ModifyUserReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -487,11 +450,8 @@ grpc::Status CraneCtldSecureServiceImpl::ModifyQos( grpc::ServerContext *context, const crane::grpc::ModifyQosRequest *request, crane::grpc::ModifyQosReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_INVALID_UID); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -575,11 +535,8 @@ grpc::Status CraneCtldSecureServiceImpl::QueryUserInfo( const crane::grpc::QueryUserInfoRequest *request, crane::grpc::QueryUserInfoReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -641,11 +598,8 @@ grpc::Status CraneCtldSecureServiceImpl::QueryQosInfo( std::unordered_map res_qos_map; auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -678,11 +632,8 @@ grpc::Status CraneCtldSecureServiceImpl::DeleteAccount( const crane::grpc::DeleteAccountRequest *request, crane::grpc::DeleteAccountReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -700,11 +651,8 @@ grpc::Status CraneCtldSecureServiceImpl::DeleteUser( grpc::ServerContext *context, const crane::grpc::DeleteUserRequest *request, crane::grpc::DeleteUserReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); auto res = @@ -723,11 +671,8 @@ grpc::Status CraneCtldSecureServiceImpl::DeleteQos( grpc::ServerContext *context, const crane::grpc::DeleteQosRequest *request, crane::grpc::DeleteQosReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); @@ -747,11 +692,8 @@ grpc::Status CraneCtldSecureServiceImpl::BlockAccountOrUser( const crane::grpc::BlockAccountOrUserRequest *request, crane::grpc::BlockAccountOrUserReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); AccountManager::CraneExpected res; @@ -784,11 +726,8 @@ grpc::Status CraneCtldSecureServiceImpl::ResetUserCredential( const crane::grpc::ResetUserCredentialRequest *request, crane::grpc::ResetUserCredentialReply *response) { auto extract_result = CheckCertAllowedAndExtractUIDFromCert_(context); - if (!extract_result) { - response->set_ok(false); - response->set_reason(crane::grpc::ErrCode::ERR_PERMISSION_USER); + if (!extract_result) return grpc::Status(grpc::UNAUTHENTICATED, "Authentication failed"); - } uint32_t uid = extract_result.value(); diff --git a/src/Utilities/VaultClient/VaultClient.cpp b/src/Utilities/VaultClient/VaultClient.cpp index 3872ee70e..adffcf069 100644 --- a/src/Utilities/VaultClient/VaultClient.cpp +++ b/src/Utilities/VaultClient/VaultClient.cpp @@ -87,8 +87,8 @@ std::expected VaultClient::Sign( bool VaultClient::RevokeCert(const std::string& serial_number) { try { - auto response = pki_external_->revokeCertificate( - Vault::Parameters{{"serial_number", serial_number}}); + auto response = + RevokeCertificate_(Vault::Parameters{{"serial_number", serial_number}}); if (!response) return false; } catch (const std::exception& e) { return false; @@ -129,6 +129,14 @@ std::optional VaultClient::ListRevokeCertificate_() { Vault::Path{"certs/revoked"})); } +std::optional VaultClient::RevokeCertificate_( + const Vault::Parameters& parameters) { + return post_( + *root_client_, + GetPkiUrl_(Vault::SecretMount{"pki_external"}, Vault::Path{"revoke"}), + parameters); +} + std::optional VaultClient::list_(const Vault::Client& client, const Vault::Url& url) { if (!client.is_authenticated()) { @@ -154,6 +162,40 @@ std::optional VaultClient::list_(const Vault::Client& client, return std::nullopt; } +// Revoking a non-existent certificate does not throw an error. +std::optional VaultClient::post_( + const Vault::Client& client, const Vault::Url& url, + const Vault::Parameters& parameters) { + if (!client.is_authenticated()) { + return std::nullopt; + } + + nlohmann::json json = create_json(parameters); + + auto response = client.getHttpClient().post( + url, client.getToken(), client.getNamespace(), json.dump()); + + if (Vault::HttpClient::is_success(response)) { + return response.value().body.value(); + } + // Do not return an error when revoke not found certificate. + if (response) { + auto jsonResponse = nlohmann::json::parse(response.value().body.value()); + const auto res_err = jsonResponse["errors"]; + static const LazyRE2 pattern( + R"(certificate with serial (\S+) not found\.)"); + if (jsonResponse.contains("errors") && res_err.is_array() && + res_err.size() == 1 && RE2::FullMatch(res_err[0], *pattern)) { + CRANE_DEBUG("{}", res_err[0]); + return ""; + } else { + client.getHttpClient().handleResponseError(response.value()); + } + } + + return std::nullopt; +} + Vault::Url VaultClient::GetPkiUrl_(const Vault::SecretMount secret_mount, const Vault::Path& path) const { return GetUrl_("/v1/" + secret_mount, @@ -166,4 +208,25 @@ Vault::Url VaultClient::GetUrl_(const std::string& base, base + path}; } +nlohmann::json VaultClient::create_json(const Vault::Parameters& parameters) { + nlohmann::json json = nlohmann::json::object(); + for (auto& [key, value] : parameters) { + if (std::holds_alternative(value)) { + json[key] = std::get(value); + } else if (std::holds_alternative(value)) { + json[key] = std::get(value); + } else if (std::holds_alternative>(value)) { + json[key] = std::get>(value); + } else if (std::holds_alternative(value)) { + auto map = std::get(value); + nlohmann::json j; + for (auto& [map_key, map_value] : map) { + j[map_key] = map_value; + } + json[key] = j; + } + } + return json; +} + } // namespace vault \ No newline at end of file diff --git a/src/Utilities/VaultClient/include/crane/VaultClient.h b/src/Utilities/VaultClient/include/crane/VaultClient.h index ce743d35b..7fa67d060 100644 --- a/src/Utilities/VaultClient/include/crane/VaultClient.h +++ b/src/Utilities/VaultClient/include/crane/VaultClient.h @@ -61,14 +61,23 @@ class VaultClient { std::optional ListRevokeCertificate_(); + std::optional RevokeCertificate_( + const Vault::Parameters& parameters); + std::optional list_(const Vault::Client& client, const Vault::Url& url); + std::optional post_(const Vault::Client& client, + const Vault::Url& url, + const Vault::Parameters& parameters); + Vault::Url GetUrl_(const std::string& base, const Vault::Path& path) const; Vault::Url GetPkiUrl_(const Vault::SecretMount secret_mount, const Vault::Path& path) const; + nlohmann::json create_json(const Vault::Parameters& parameters); + std::unique_ptr root_client_; std::unique_ptr pki_root_; std::unique_ptr pki_external_; From fee150fc4d4e2561fc2410225241126df6e7c868 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Mon, 17 Feb 2025 17:35:43 +0800 Subject: [PATCH 60/62] fix: revoke empty serial_number --- src/CraneCtld/AccountManager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index fb7001549..6e4951f51 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1857,7 +1857,8 @@ AccountManager::CraneExpected AccountManager::DeleteUser_( } } - if (res_user.deleted && !g_vault_client->RevokeCert(res_user.serial_number)) + if (res_user.deleted && res_user.serial_number != "" && + !g_vault_client->RevokeCert(res_user.serial_number)) return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); mongocxx::client_session::with_transaction_cb callback = From 5c98d941310e303c9efb049770fc7320fa719807 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 20 Feb 2025 09:34:59 +0800 Subject: [PATCH 61/62] refactor --- src/CraneCtld/AccountManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index 6e4951f51..b5ea5d248 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -1857,7 +1857,7 @@ AccountManager::CraneExpected AccountManager::DeleteUser_( } } - if (res_user.deleted && res_user.serial_number != "" && + if (res_user.deleted && !res_user.serial_number.empty() && !g_vault_client->RevokeCert(res_user.serial_number)) return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); From dbf59f268e8256034301f885fb0c99b4fccde979 Mon Sep 17 00:00:00 2001 From: huerni <47264950+huerni@users.noreply.github.com> Date: Thu, 20 Feb 2025 15:27:39 +0800 Subject: [PATCH 62/62] feat: add log --- src/CraneCtld/AccountManager.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CraneCtld/AccountManager.cpp b/src/CraneCtld/AccountManager.cpp index b5ea5d248..65be52d40 100644 --- a/src/CraneCtld/AccountManager.cpp +++ b/src/CraneCtld/AccountManager.cpp @@ -19,6 +19,7 @@ #include "AccountManager.h" #include "AccountMetaContainer.h" +#include "crane/Logger.h" #include "protos/PublicDefs.pb.h" #include "range/v3/algorithm/contains.hpp" @@ -1095,6 +1096,8 @@ AccountManager::CraneExpected AccountManager::ResetUserCertificate( if (!user->serial_number.empty() && !g_vault_client->RevokeCert(user->serial_number)) if (!force) return std::unexpected(CraneErrCode::ERR_REVOKE_CERTIFICATE); + CRANE_DEBUG("Reset User {} Certificate {}", user->name, + user->serial_number); } // Save the serial number in the database.