diff --git a/src/llmq/quorums.cpp b/src/llmq/quorums.cpp
index a26e38053f804..8dc8b17134de0 100644
--- a/src/llmq/quorums.cpp
+++ b/src/llmq/quorums.cpp
@@ -190,21 +190,29 @@ void CQuorumManager::EnsureQuorumConnections(Consensus::LLMQType llmqType, const
         }
 
         if (!g_connman->HasMasternodeQuorumNodes(llmqType, quorum->qc.quorumHash)) {
-            std::map<CService, uint256> connections;
+            std::set<uint256> connections;
             if (quorum->IsMember(myProTxHash)) {
                 connections = CLLMQUtils::GetQuorumConnections(llmqType, quorum->qc.quorumHash, myProTxHash);
             } else {
                 auto cindexes = CLLMQUtils::CalcDeterministicWatchConnections(llmqType, quorum->qc.quorumHash, quorum->members.size(), 1);
                 for (auto idx : cindexes) {
-                    connections.emplace(quorum->members[idx]->pdmnState->addr, quorum->members[idx]->proTxHash);
+                    connections.emplace(quorum->members[idx]->proTxHash);
                 }
             }
             if (!connections.empty()) {
-                std::string debugMsg = strprintf("CQuorumManager::%s -- adding masternodes quorum connections for quorum %s:\n", __func__, quorum->qc.quorumHash.ToString());
-                for (auto& c : connections) {
-                    debugMsg += strprintf("  %s\n", c.first.ToString(false));
+                if (LogAcceptCategory("llmq")) {
+                    auto mnList = deterministicMNManager->GetListAtChainTip();
+                    std::string debugMsg = strprintf("CQuorumManager::%s -- adding masternodes quorum connections for quorum %s:\n", __func__, quorum->qc.quorumHash.ToString());
+                    for (auto& c : connections) {
+                        auto dmn = mnList.GetValidMN(c);
+                        if (!dmn) {
+                            debugMsg += strprintf("  %s (not in valid MN set anymore)\n", c.ToString());
+                        } else {
+                            debugMsg += strprintf("  %s (%s)\n", c.ToString(), dmn->pdmnState->addr.ToString(false));
+                        }
+                    }
+                    LogPrint("llmq", debugMsg);
                 }
-                LogPrint("llmq", debugMsg);
                 g_connman->AddMasternodeQuorumNodes(llmqType, quorum->qc.quorumHash, connections);
             }
         }
diff --git a/src/llmq/quorums_dkgsessionhandler.cpp b/src/llmq/quorums_dkgsessionhandler.cpp
index 4cd69f4b70451..d20c25862906b 100644
--- a/src/llmq/quorums_dkgsessionhandler.cpp
+++ b/src/llmq/quorums_dkgsessionhandler.cpp
@@ -468,21 +468,29 @@ void CDKGSessionHandler::HandleDKGRound()
     });
 
     if (curSession->AreWeMember() || GetBoolArg("-watchquorums", DEFAULT_WATCH_QUORUMS)) {
-        std::map<CService, uint256> connections;
+        std::set<uint256> connections;
         if (curSession->AreWeMember()) {
             connections = CLLMQUtils::GetQuorumConnections(params.type, curQuorumHash, curSession->myProTxHash);
         } else {
             auto cindexes = CLLMQUtils::CalcDeterministicWatchConnections(params.type, curQuorumHash, curSession->members.size(), 1);
             for (auto idx : cindexes) {
-                connections.emplace(curSession->members[idx]->dmn->pdmnState->addr, curSession->members[idx]->dmn->proTxHash);
+                connections.emplace(curSession->members[idx]->dmn->proTxHash);
             }
         }
         if (!connections.empty()) {
-            std::string debugMsg = strprintf("CDKGSessionManager::%s -- adding masternodes quorum connections for quorum %s:\n", __func__, curSession->quorumHash.ToString());
-            for (const auto& c : connections) {
-                debugMsg += strprintf("  %s\n", c.first.ToString(false));
+            if (LogAcceptCategory("llmq-dkg")) {
+                std::string debugMsg = strprintf("CDKGSessionManager::%s -- adding masternodes quorum connections for quorum %s:\n", __func__, curSession->quorumHash.ToString());
+                auto mnList = deterministicMNManager->GetListAtChainTip();
+                for (const auto& c : connections) {
+                    auto dmn = mnList.GetValidMN(c);
+                    if (!dmn) {
+                        debugMsg += strprintf("  %s (not in valid MN set anymore)\n", c.ToString());
+                    } else {
+                        debugMsg += strprintf("  %s (%s)\n", c.ToString(), dmn->pdmnState->addr.ToString(false));
+                    }
+                }
+                LogPrint("llmq-dkg", debugMsg);
             }
-            LogPrint("llmq-dkg", debugMsg);
             g_connman->AddMasternodeQuorumNodes(params.type, curQuorumHash, connections);
         }
     }
diff --git a/src/llmq/quorums_utils.cpp b/src/llmq/quorums_utils.cpp
index 5d1de8791d8cf..338b01bf54c78 100644
--- a/src/llmq/quorums_utils.cpp
+++ b/src/llmq/quorums_utils.cpp
@@ -41,12 +41,12 @@ uint256 CLLMQUtils::BuildSignHash(Consensus::LLMQType llmqType, const uint256& q
     return h.GetHash();
 }
 
-std::map<CService, uint256> CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, const uint256& blockHash, const uint256& forMember)
+std::set<uint256> CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, const uint256& blockHash, const uint256& forMember)
 {
     auto& params = Params().GetConsensus().llmqs.at(llmqType);
 
     auto mns = GetAllQuorumMembers(llmqType, blockHash);
-    std::map<CService, uint256> result;
+    std::set<uint256> result;
     for (size_t i = 0; i < mns.size(); i++) {
         auto& dmn = mns[i];
         if (dmn->proTxHash == forMember) {
@@ -62,7 +62,7 @@ std::map<CService, uint256> CLLMQUtils::GetQuorumConnections(Consensus::LLMQType
                 if (otherDmn == dmn) {
                     continue;
                 }
-                result.emplace(otherDmn->pdmnState->addr, otherDmn->proTxHash);
+                result.emplace(otherDmn->proTxHash);
                 gap <<= 1;
                 k++;
             }
diff --git a/src/llmq/quorums_utils.h b/src/llmq/quorums_utils.h
index a53afdc874bed..a8c5ba7924bbf 100644
--- a/src/llmq/quorums_utils.h
+++ b/src/llmq/quorums_utils.h
@@ -31,7 +31,7 @@ class CLLMQUtils
         return BuildSignHash((Consensus::LLMQType)s.llmqType, s.quorumHash, s.id, s.msgHash);
     }
 
-    static std::map<CService, uint256> GetQuorumConnections(Consensus::LLMQType llmqType, const uint256& blockHash, const uint256& forMember);
+    static std::set<uint256> GetQuorumConnections(Consensus::LLMQType llmqType, const uint256& blockHash, const uint256& forMember);
     static std::set<size_t> CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const uint256& blockHash, size_t memberCount, size_t connectionCount);
 
     static bool IsQuorumActive(Consensus::LLMQType llmqType, const uint256& quorumHash);
diff --git a/src/net.cpp b/src/net.cpp
index 7c39ea65c6a2e..be2643dac25f0 100644
--- a/src/net.cpp
+++ b/src/net.cpp
@@ -2070,8 +2070,7 @@ void CConnman::ThreadOpenMasternodeConnections()
 
             std::vector<CService> pending;
             for (const auto& group : masternodeQuorumNodes) {
-                for (const auto& p : group.second) {
-                    const auto& proRegTxHash = p.second;
+                for (const auto& proRegTxHash : group.second) {
                     auto dmn = mnList.GetValidMN(proRegTxHash);
                     if (!dmn) {
                         continue;
@@ -2725,14 +2724,14 @@ bool CConnman::AddPendingMasternode(const CService& service)
     return true;
 }
 
-bool CConnman::AddMasternodeQuorumNodes(Consensus::LLMQType llmqType, const uint256& quorumHash, const std::map<CService, uint256>& addresses)
+bool CConnman::AddMasternodeQuorumNodes(Consensus::LLMQType llmqType, const uint256& quorumHash, const std::set<uint256>& proTxHashes)
 {
     LOCK(cs_vPendingMasternodes);
     auto it = masternodeQuorumNodes.find(std::make_pair(llmqType, quorumHash));
     if (it != masternodeQuorumNodes.end()) {
         return false;
     }
-    masternodeQuorumNodes.emplace(std::make_pair(llmqType, quorumHash), addresses);
+    masternodeQuorumNodes.emplace(std::make_pair(llmqType, quorumHash), proTxHashes);
     return true;
 }
 
@@ -2762,18 +2761,14 @@ std::set<NodeId> CConnman::GetMasternodeQuorumNodes(Consensus::LLMQType llmqType
     if (it == masternodeQuorumNodes.end()) {
         return {};
     }
-    std::set<uint256> proRegTxHashes;
-    for (auto& p : it->second) {
-        proRegTxHashes.emplace(p.second);
-    }
+    const auto& proRegTxHashes = it->second;
 
     std::set<NodeId> nodes;
     for (const auto pnode : vNodes) {
         if (pnode->fDisconnect) {
             continue;
         }
-        if (!pnode->qwatch && !it->second.count(pnode->addr) &&
-            (pnode->verifiedProRegTxHash.IsNull() || !proRegTxHashes.count(pnode->verifiedProRegTxHash))) {
+        if (!pnode->qwatch && (pnode->verifiedProRegTxHash.IsNull() || !proRegTxHashes.count(pnode->verifiedProRegTxHash))) {
             continue;
         }
         nodes.emplace(pnode->id);
@@ -2789,13 +2784,27 @@ void CConnman::RemoveMasternodeQuorumNodes(Consensus::LLMQType llmqType, const u
 
 bool CConnman::IsMasternodeQuorumNode(const CNode* pnode)
 {
+    // Let's see if this is an outgoing connection to an address that is known to be a masternode
+    // We however only need to know this if the node did not authenticate itself as a MN yet
+    uint256 assumedProTxHash;
+    if (pnode->verifiedProRegTxHash.IsNull() && !pnode->fInbound) {
+        auto mnList = deterministicMNManager->GetListAtChainTip();
+        auto dmn = mnList.GetValidMNByService(pnode->addr);
+        if (dmn == nullptr) {
+            // This is definitely not a masternode
+            return false;
+        }
+        assumedProTxHash = dmn->proTxHash;
+    }
+
     LOCK(cs_vPendingMasternodes);
     for (const auto& p : masternodeQuorumNodes) {
-        for (const auto& p2 : p.second) {
-            if (p2.first == (CService)pnode->addr) {
+        if (!pnode->verifiedProRegTxHash.IsNull()) {
+            if (p.second.count(pnode->verifiedProRegTxHash)) {
                 return true;
             }
-            if (!pnode->verifiedProRegTxHash.IsNull() && p2.second == pnode->verifiedProRegTxHash) {
+        } else if (!assumedProTxHash.IsNull()) {
+            if (p.second.count(assumedProTxHash)) {
                 return true;
             }
         }
diff --git a/src/net.h b/src/net.h
index 90b2c719620ff..c383d0bc2a9c4 100644
--- a/src/net.h
+++ b/src/net.h
@@ -356,7 +356,7 @@ class CConnman
     std::vector<AddedNodeInfo> GetAddedNodeInfo();
 
     bool AddPendingMasternode(const CService& addr);
-    bool AddMasternodeQuorumNodes(Consensus::LLMQType llmqType, const uint256& quorumHash, const std::map<CService, uint256>& addresses);
+    bool AddMasternodeQuorumNodes(Consensus::LLMQType llmqType, const uint256& quorumHash, const std::set<uint256>& proTxHashes);
     bool HasMasternodeQuorumNodes(Consensus::LLMQType llmqType, const uint256& quorumHash);
     std::set<uint256> GetMasternodeQuorums(Consensus::LLMQType llmqType);
     // also returns QWATCH nodes
@@ -492,7 +492,7 @@ class CConnman
     std::vector<std::string> vAddedNodes;
     CCriticalSection cs_vAddedNodes;
     std::vector<CService> vPendingMasternodes;
-    std::map<std::pair<Consensus::LLMQType, uint256>, std::map<CService, uint256>> masternodeQuorumNodes; // protected by cs_vPendingMasternodes
+    std::map<std::pair<Consensus::LLMQType, uint256>, std::set<uint256>> masternodeQuorumNodes; // protected by cs_vPendingMasternodes
     mutable CCriticalSection cs_vPendingMasternodes;
     std::vector<CNode*> vNodes;
     std::list<CNode*> vNodesDisconnected;