Skip to content
This repository has been archived by the owner on Sep 5, 2023. It is now read-only.

Commit

Permalink
examples: make example-09 use seperate receive CQ
Browse files Browse the repository at this point in the history
Signed-off-by: Xiao Yang <[email protected]>
  • Loading branch information
yangx-jy committed Jun 30, 2021
1 parent 0f37fa2 commit ba2b96a
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 14 deletions.
47 changes: 41 additions & 6 deletions examples/09-flush-to-persistent-GPSPM/client.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2020, Intel Corporation */
/* Copyright 2021, Fujitsu */

/*
* client.c -- a client of the flush-to-persistent-GPSPM example
Expand Down Expand Up @@ -181,8 +182,15 @@ main(int argc, char *argv[])
if ((ret = client_peer_via_address(addr, &peer)))
goto err_free;

struct rpma_conn_cfg *cfg = NULL;
if ((ret = rpma_conn_cfg_new(&cfg)))
goto err_free;

if ((ret = rpma_conn_cfg_set_rcq_size(cfg, RCQ_SIZE)))
goto err_cfg_delete;

/* establish a new connection to a server listening at addr:port */
if ((ret = client_connect(peer, addr, port, NULL, NULL, &conn)))
if ((ret = client_connect(peer, addr, port, cfg, NULL, &conn)))
goto err_peer_delete;

/* register the memory RDMA write */
Expand Down Expand Up @@ -250,16 +258,40 @@ main(int argc, char *argv[])

/* send the flush message */
if ((ret = rpma_send(conn, msg_mr, SEND_OFFSET, flush_req_size,
RPMA_F_COMPLETION_ON_ERROR, NULL)))
RPMA_F_COMPLETION_ALWAYS, NULL)))
goto err_mr_remote_delete;

/* wait for the completion to be ready */
if ((ret = rpma_conn_completion_wait(conn)))
/* wait for the sending completion to be ready */
struct rpma_cq *cq = NULL;
if ((ret = rpma_conn_get_cq(conn, &cq)))
goto err_mr_remote_delete;
if ((ret = rpma_cq_wait(cq)))
goto err_mr_remote_delete;
if ((ret = rpma_conn_completion_get(conn, &cmpl)))
if ((ret = rpma_cq_get_completion(cq, &cmpl)))
goto err_mr_remote_delete;

/* validate the completion */
/* validate the sending completion */
if (cmpl.op_status != IBV_WC_SUCCESS)
goto err_mr_remote_delete;
if (cmpl.op != RPMA_OP_SEND) {
(void) fprintf(stderr,
"unexpected cmpl.op value "
"(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n",
(uintptr_t)cmpl.op,
(uintptr_t)RPMA_OP_SEND);
goto err_mr_remote_delete;
}

/* wait for the receive completion to be ready */
struct rpma_cq *rcq = NULL;
if ((ret = rpma_conn_get_rcq(conn, &rcq)))
goto err_mr_remote_delete;
if ((ret = rpma_cq_wait(rcq)))
goto err_mr_remote_delete;
if ((ret = rpma_cq_get_completion(rcq, &cmpl)))
goto err_mr_remote_delete;

/* validate the receive completion */
if (cmpl.op_status != IBV_WC_SUCCESS)
goto err_mr_remote_delete;
if (cmpl.op != RPMA_OP_RECV) {
Expand Down Expand Up @@ -316,6 +348,9 @@ main(int argc, char *argv[])
/* delete the peer */
(void) rpma_peer_delete(&peer);

err_cfg_delete:
(void) rpma_conn_cfg_delete(&cfg);

err_free:
free(msg_ptr);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,6 @@
#define SEND_OFFSET 0
#define RECV_OFFSET (SEND_OFFSET + MSG_SIZE_MAX)

#define RCQ_SIZE 1

#endif /* EXAMPLES_FLUSH_2_PMEM_GPSPM */
45 changes: 37 additions & 8 deletions examples/09-flush-to-persistent-GPSPM/server.c
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2020-2021, Intel Corporation */
/* Copyright 2021, Fujitsu */

/*
* server.c -- a server of the flush-to-persistent-GPSPM example
Expand Down Expand Up @@ -194,6 +195,13 @@ main(int argc, char *argv[])
if ((ret = rpma_mr_get_descriptor(mr, &data.descriptors[0])))
goto err_mr_dereg;

struct rpma_conn_cfg *cfg = NULL;
if ((ret = rpma_conn_cfg_new(&cfg)))
goto err_mr_dereg;

if ((ret = rpma_conn_cfg_set_rcq_size(cfg, RCQ_SIZE)))
goto err_cfg_delete;

/*
* Wait for an incoming connection request, accept it and wait for its
* establishment.
Expand All @@ -203,7 +211,7 @@ main(int argc, char *argv[])
pdata.len = sizeof(struct common_data);

/* receive an incoming connection request */
if ((ret = rpma_ep_next_conn_req(ep, NULL, &req)))
if ((ret = rpma_ep_next_conn_req(ep, cfg, &req)))
goto err_mr_dereg;

/* prepare buffer for a flush request */
Expand All @@ -228,12 +236,27 @@ main(int argc, char *argv[])
if (ret)
goto err_conn_delete;

/* wait for the completion to be ready */
if ((ret = rpma_conn_completion_wait(conn)))
/* wait for the receive completion to be ready */
struct rpma_cq *rcq = NULL;
if ((ret = rpma_conn_get_rcq(conn, &rcq)))
goto err_conn_delete;
if ((ret = rpma_cq_wait(rcq)))
goto err_conn_delete;
if ((ret = rpma_conn_completion_get(conn, &cmpl)))
if ((ret = rpma_cq_get_completion(rcq, &cmpl)))
goto err_conn_delete;

/* validate the receive completion */
if (cmpl.op_status != IBV_WC_SUCCESS)
goto err_conn_delete;
if (cmpl.op != RPMA_OP_RECV) {
(void) fprintf(stderr,
"unexpected cmpl.op value "
"(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n",
(uintptr_t)cmpl.op,
(uintptr_t)RPMA_OP_RECV);
goto err_conn_delete;
}

/* unpack a flush request from the received buffer */
flush_req = gpspm_flush_request__unpack(NULL, cmpl.byte_len, recv_ptr);
if (flush_req == NULL) {
Expand Down Expand Up @@ -271,13 +294,16 @@ main(int argc, char *argv[])
RPMA_F_COMPLETION_ALWAYS, NULL)))
goto err_conn_delete;

/* wait for the completion to be ready */
if ((ret = rpma_conn_completion_wait(conn)))
/* wait for the sending completion to be ready */
struct rpma_cq *cq = NULL;
if ((ret = rpma_conn_get_cq(conn, &cq)))
goto err_conn_delete;
if ((ret = rpma_cq_wait(cq)))
goto err_conn_delete;
if ((ret = rpma_conn_completion_get(conn, &cmpl)))
if ((ret = rpma_cq_get_completion(cq, &cmpl)))
goto err_conn_delete;

/* validate the completion */
/* validate the sending completion */
if (cmpl.op_status != IBV_WC_SUCCESS)
goto err_conn_delete;
if (cmpl.op != RPMA_OP_SEND) {
Expand Down Expand Up @@ -305,6 +331,9 @@ main(int argc, char *argv[])
err_req_delete:
(void) rpma_conn_req_delete(&req);

err_cfg_delete:
(void) rpma_conn_cfg_delete(&cfg);

err_mr_dereg:
(void) rpma_mr_dereg(&msg_mr);
(void) rpma_mr_dereg(&mr);
Expand Down

0 comments on commit ba2b96a

Please sign in to comment.