diff --git a/examples/09-flush-to-persistent-GPSPM/client.c b/examples/09-flush-to-persistent-GPSPM/client.c index 073cf01d41..6ad343f83b 100644 --- a/examples/09-flush-to-persistent-GPSPM/client.c +++ b/examples/09-flush-to-persistent-GPSPM/client.c @@ -185,10 +185,17 @@ main(int argc, char *argv[]) if ((ret = client_peer_via_address(addr, &peer))) goto err_free; - /* establish a new connection to a server listening at addr:port */ - if ((ret = client_connect(peer, addr, port, NULL, NULL, &conn))) + struct rpma_conn_cfg *cfg = NULL; + if ((ret = rpma_conn_cfg_new(&cfg))) goto err_peer_delete; + if ((ret = rpma_conn_cfg_set_rcq_size(cfg, RCQ_SIZE))) + goto err_cfg_delete; + + /* establish a new connection to a server listening at addr:port */ + if ((ret = client_connect(peer, addr, port, cfg, NULL, &conn))) + goto err_cfg_delete; + /* register the memory RDMA write */ if ((ret = rpma_mr_reg(peer, mr_ptr, mr_size, RPMA_MR_USAGE_WRITE_SRC, &src_mr))) @@ -254,16 +261,46 @@ main(int argc, char *argv[]) /* send the flush message */ if ((ret = rpma_send(conn, msg_mr, SEND_OFFSET, flush_req_size, - RPMA_F_COMPLETION_ON_ERROR, NULL))) + RPMA_F_COMPLETION_ALWAYS, NULL))) goto err_mr_remote_delete; - /* wait for the completion to be ready */ - if ((ret = rpma_conn_completion_wait(conn))) + /* wait for the send completion to be ready */ + struct rpma_cq *cq = NULL; + if ((ret = rpma_conn_get_cq(conn, &cq))) + goto err_mr_remote_delete; + if ((ret = rpma_cq_wait(cq))) goto err_mr_remote_delete; - if ((ret = rpma_conn_completion_get(conn, &cmpl))) + if ((ret = rpma_cq_get_completion(cq, &cmpl))) goto err_mr_remote_delete; - /* validate the completion */ + /* validate the send completion */ + if (cmpl.op_status != IBV_WC_SUCCESS) { + ret = -1; + (void) fprintf(stderr, "rpma_send() failed: %s\n", + ibv_wc_status_str(cmpl.op_status)); + goto err_mr_remote_delete; + } + + if (cmpl.op != RPMA_OP_SEND) { + ret = -1; + (void) fprintf(stderr, + "unexpected cmpl.op value " + "(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n", + (uintptr_t)cmpl.op, + (uintptr_t)RPMA_OP_SEND); + goto err_mr_remote_delete; + } + + /* wait for the receive completion to be ready */ + struct rpma_cq *rcq = NULL; + if ((ret = rpma_conn_get_rcq(conn, &rcq))) + goto err_mr_remote_delete; + if ((ret = rpma_cq_wait(rcq))) + goto err_mr_remote_delete; + if ((ret = rpma_cq_get_completion(rcq, &cmpl))) + goto err_mr_remote_delete; + + /* validate the receive completion */ if (cmpl.op_status != IBV_WC_SUCCESS) { ret = -1; (void) fprintf(stderr, "rpma_recv() failed: %s\n", @@ -322,6 +359,9 @@ main(int argc, char *argv[]) err_conn_disconnect: (void) common_disconnect_and_wait_for_conn_close(&conn); +err_cfg_delete: + (void) rpma_conn_cfg_delete(&cfg); + err_peer_delete: /* delete the peer */ (void) rpma_peer_delete(&peer); diff --git a/examples/09-flush-to-persistent-GPSPM/flush-to-persistent-GPSPM.h b/examples/09-flush-to-persistent-GPSPM/flush-to-persistent-GPSPM.h index db390ec0ae..941b6bf5b1 100644 --- a/examples/09-flush-to-persistent-GPSPM/flush-to-persistent-GPSPM.h +++ b/examples/09-flush-to-persistent-GPSPM/flush-to-persistent-GPSPM.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright 2020, Intel Corporation */ +/* Copyright 2021, Fujitsu */ /* * flush-to-persistent-GSPSPM.h -- a common declarations for the 09 example @@ -13,4 +14,6 @@ #define SEND_OFFSET 0 #define RECV_OFFSET (SEND_OFFSET + MSG_SIZE_MAX) +#define RCQ_SIZE 1 + #endif /* EXAMPLES_FLUSH_2_PMEM_GPSPM */ diff --git a/examples/09-flush-to-persistent-GPSPM/server.c b/examples/09-flush-to-persistent-GPSPM/server.c index 624cc4fd53..7719fc9f1c 100644 --- a/examples/09-flush-to-persistent-GPSPM/server.c +++ b/examples/09-flush-to-persistent-GPSPM/server.c @@ -198,6 +198,13 @@ main(int argc, char *argv[]) if ((ret = rpma_mr_get_descriptor(mr, &data.descriptors[0]))) goto err_mr_dereg; + struct rpma_conn_cfg *cfg = NULL; + if ((ret = rpma_conn_cfg_new(&cfg))) + goto err_mr_dereg; + + if ((ret = rpma_conn_cfg_set_rcq_size(cfg, RCQ_SIZE))) + goto err_cfg_delete; + /* * Wait for an incoming connection request, accept it and wait for its * establishment. @@ -207,7 +214,7 @@ main(int argc, char *argv[]) pdata.len = sizeof(struct common_data); /* receive an incoming connection request */ - if ((ret = rpma_ep_next_conn_req(ep, NULL, &req))) + if ((ret = rpma_ep_next_conn_req(ep, cfg, &req))) goto err_mr_dereg; /* prepare buffer for a flush request */ @@ -232,12 +239,16 @@ main(int argc, char *argv[]) if (ret) goto err_conn_delete; - /* wait for the completion to be ready */ - if ((ret = rpma_conn_completion_wait(conn))) + /* wait for the receive completion to be ready */ + struct rpma_cq *rcq = NULL; + if ((ret = rpma_conn_get_rcq(conn, &rcq))) + goto err_conn_delete; + if ((ret = rpma_cq_wait(rcq))) goto err_conn_delete; - if ((ret = rpma_conn_completion_get(conn, &cmpl))) + if ((ret = rpma_cq_get_completion(rcq, &cmpl))) goto err_conn_delete; + /* validate the receive completion */ if (cmpl.op_status != IBV_WC_SUCCESS) { ret = -1; (void) fprintf(stderr, "rpma_recv() failed: %s\n", @@ -245,6 +256,16 @@ main(int argc, char *argv[]) goto err_conn_delete; } + if (cmpl.op != RPMA_OP_RECV) { + ret = -1; + (void) fprintf(stderr, + "unexpected cmpl.op value " + "(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n", + (uintptr_t)cmpl.op, + (uintptr_t)RPMA_OP_RECV); + goto err_conn_delete; + } + /* unpack a flush request from the received buffer */ flush_req = gpspm_flush_request__unpack(NULL, cmpl.byte_len, recv_ptr); if (flush_req == NULL) { @@ -282,13 +303,16 @@ main(int argc, char *argv[]) RPMA_F_COMPLETION_ALWAYS, NULL))) goto err_conn_delete; - /* wait for the completion to be ready */ - if ((ret = rpma_conn_completion_wait(conn))) + /* wait for the send completion to be ready */ + struct rpma_cq *cq = NULL; + if ((ret = rpma_conn_get_cq(conn, &cq))) + goto err_conn_delete; + if ((ret = rpma_cq_wait(cq))) goto err_conn_delete; - if ((ret = rpma_conn_completion_get(conn, &cmpl))) + if ((ret = rpma_cq_get_completion(cq, &cmpl))) goto err_conn_delete; - /* validate the completion */ + /* validate the send completion */ if (cmpl.op_status != IBV_WC_SUCCESS) { ret = -1; (void) fprintf(stderr, "rpma_send() failed: %s\n", @@ -322,6 +346,9 @@ main(int argc, char *argv[]) err_req_delete: (void) rpma_conn_req_delete(&req); +err_cfg_delete: + (void) rpma_conn_cfg_delete(&cfg); + err_mr_dereg: (void) rpma_mr_dereg(&msg_mr); (void) rpma_mr_dereg(&mr);