in prov/gni/src/gnix_msg.c [2973:3149]
static int _gnix_send_req(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_nic *nic;
struct gnix_fid_ep *ep;
struct gnix_tx_descriptor *tdesc;
gni_return_t status;
int rc;
int rendezvous = !!(req->msg.send_flags & GNIX_MSG_RENDEZVOUS);
int hdr_len, data_len;
void *hdr, *data;
int tag;
int inject_err = _gnix_req_inject_smsg_err(req);
ep = req->gnix_ep;
assert(ep != NULL);
nic = ep->nic;
assert(nic != NULL);
rc = _gnix_nic_tx_alloc(nic, &tdesc);
if (rc != FI_SUCCESS) {
GNIX_DEBUG(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
assert(rc == FI_SUCCESS);
if (OFI_UNLIKELY(rendezvous)) {
switch(req->type) {
case GNIX_FAB_RQ_SEND:
case GNIX_FAB_RQ_TSEND:
assert(req->msg.send_md);
tag = GNIX_SMSG_T_RNDZV_START;
tdesc->rndzv_start_hdr.flags = req->msg.send_flags;
tdesc->rndzv_start_hdr.imm = req->msg.imm;
tdesc->rndzv_start_hdr.msg_tag = req->msg.tag;
tdesc->rndzv_start_hdr.mdh = req->msg.send_info[0].mem_hndl;
tdesc->rndzv_start_hdr.addr = req->msg.send_info[0].send_addr;
tdesc->rndzv_start_hdr.len = req->msg.send_info[0].send_len;
tdesc->rndzv_start_hdr.req_addr = (uint64_t)req;
if (req->msg.send_info[0].send_addr & GNI_READ_ALIGN_MASK) {
tdesc->rndzv_start_hdr.head =
*(uint32_t *)(req->msg.send_info[0].send_addr &
~GNI_READ_ALIGN_MASK);
GNIX_DEBUG(FI_LOG_EP_DATA,
"Sending %d unaligned head bytes (%x)\n",
GNI_READ_ALIGN -
(req->msg.send_info[0].send_addr &
GNI_READ_ALIGN_MASK),
tdesc->rndzv_start_hdr.head);
}
if ((req->msg.send_info[0].send_addr +
req->msg.send_info[0].send_len) &
GNI_READ_ALIGN_MASK) {
tdesc->rndzv_start_hdr.tail =
*(uint32_t *)((req->msg.send_info[0].send_addr +
req->msg.send_info[0].send_len) &
~GNI_READ_ALIGN_MASK);
GNIX_DEBUG(FI_LOG_EP_DATA,
"Sending %d unaligned tail bytes (%x)\n",
(req->msg.send_info[0].send_addr +
req->msg.send_info[0].send_len) &
GNI_READ_ALIGN_MASK,
tdesc->rndzv_start_hdr.tail);
}
hdr = &tdesc->rndzv_start_hdr;
hdr_len = sizeof(tdesc->rndzv_start_hdr);
/* TODO: Unify send&sendv/recv&recvv, so data will be
* req->msg.send_info */
data = NULL;
data_len = 0;
break;
case GNIX_FAB_RQ_SENDV:
case GNIX_FAB_RQ_TSENDV:
assert(req->msg.send_md[0]);
tag = GNIX_SMSG_T_RNDZV_IOV_START;
tdesc->rndzv_iov_start_hdr.flags = req->msg.send_flags;
tdesc->rndzv_iov_start_hdr.imm = req->msg.imm;
tdesc->rndzv_iov_start_hdr.msg_tag = req->msg.tag;
tdesc->rndzv_iov_start_hdr.iov_cnt =
req->msg.send_iov_cnt;
tdesc->rndzv_iov_start_hdr.req_addr = (uint64_t) req;
tdesc->rndzv_iov_start_hdr.send_len = req->msg.cum_send_len;
/* Send data at unaligned bytes in the iov addresses
* within the data section of the control message so
* that the remote peer can pull from four byte aligned
* addresses and still transfer all the data. */
__gnix_msg_send_alignment(req);
data_len = sizeof(struct send_info_t) * req->msg.send_iov_cnt;
data = (void *) req->msg.send_info;
hdr_len = sizeof(tdesc->rndzv_iov_start_hdr);
hdr = &tdesc->rndzv_iov_start_hdr;
#if ENABLE_DEBUG
int i;
for (i = 0; i < req->msg.send_iov_cnt; i++) {
GNIX_DEBUG(FI_LOG_EP_DATA,
"data[%d].send_addr = 0x%ux, "
"data[%d].send_len = %lu, "
"data[%d].mem_hndl = %hxx\n", i,
((struct send_info_t *)data)[i].send_addr,
i,
((struct send_info_t *)data)[i].send_len,
i,
((struct send_info_t *)data)[i].mem_hndl);
}
#endif
break;
default:
GNIX_FATAL(FI_LOG_EP_DATA, "Invalid request type: %d\n", req->type);
return -FI_EINVAL;
}
} else {
tag = GNIX_SMSG_T_EGR_W_DATA;
tdesc->eager_hdr.flags = req->msg.send_flags;
tdesc->eager_hdr.imm = req->msg.imm;
tdesc->eager_hdr.msg_tag = req->msg.tag;
tdesc->eager_hdr.len = req->msg.cum_send_len;
hdr = &tdesc->eager_hdr;
hdr_len = sizeof(tdesc->eager_hdr);
data = (void *)req->msg.send_info[0].send_addr;
/* If this is not rndzv the send length should always be the
* cumulative length of all the send_info lengths */
data_len = req->msg.cum_send_len;
}
tdesc->req = req;
tdesc->completer_fn = gnix_ep_smsg_completers[tag];
COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (OFI_UNLIKELY(inject_err)) {
_gnix_nic_txd_err_inject(nic, tdesc);
status = GNI_RC_SUCCESS;
} else {
status = GNI_SmsgSendWTag(req->vc->gni_ep,
hdr, hdr_len, data, data_len,
tdesc->id, tag);
}
/*
* if this is a rendezvous message, we want to generate
* IRQ at remote peer.
* TODO: Do we need to do this for sendv?
*/
if ((status == GNI_RC_SUCCESS) &&
(tag == GNIX_SMSG_T_RNDZV_START ||
tag == GNIX_SMSG_T_RNDZV_IOV_START))
_gnix_rma_post_irq(req->vc);
COND_RELEASE(nic->requires_lock, &nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, tdesc);
GNIX_DEBUG(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, tdesc);
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
}
return gnixu_to_fi_errno(status);
}