in gloo/transport/ibverbs/pair.cc [144:192]
void Pair::connect(const std::vector<char>& bytes) {
struct ibv_qp_attr attr;
int rv;
checkErrorState();
peer_ = Address(bytes);
memset(&attr, 0, sizeof(attr));
attr.qp_state = IBV_QPS_RTR;
attr.path_mtu = IBV_MTU_1024;
attr.dest_qp_num = peer_.addr_.qpn;
attr.rq_psn = peer_.addr_.psn;
attr.max_dest_rd_atomic = 1;
attr.min_rnr_timer = 20;
attr.ah_attr.is_global = 0;
attr.ah_attr.dlid = peer_.addr_.lid;
attr.ah_attr.port_num = dev_->attr_.port;
if (peer_.addr_.ibv_gid.global.interface_id) {
attr.ah_attr.is_global = 1;
attr.ah_attr.grh.hop_limit = 1;
attr.ah_attr.grh.dgid = peer_.addr_.ibv_gid;
attr.ah_attr.grh.sgid_index = dev_->attr_.index;
}
// Move to Ready To Receive (RTR) state
rv = ibv_modify_qp(
qp_,
&attr,
IBV_QP_STATE | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN |
IBV_QP_AV | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER);
GLOO_ENFORCE_EQ(rv, 0);
memset(&attr, 0, sizeof(attr));
attr.qp_state = IBV_QPS_RTS;
attr.sq_psn = self_.addr_.psn;
attr.ah_attr.is_global = 1;
attr.timeout = 14;
attr.retry_cnt = 7;
attr.rnr_retry = 7; /* infinite */
attr.max_rd_atomic = 1;
// Move to Ready To Send (RTS) state
rv = ibv_modify_qp(
qp_,
&attr,
IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY |
IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC);
GLOO_ENFORCE_EQ(rv, 0);
}