in prov/gni/src/gnix_rma.c [964:1132]
static void __gnix_rma_more_fill_pd(struct gnix_fab_req *req,
struct gnix_tx_descriptor *txd)
{
gni_ct_put_post_descriptor_t *more_put = NULL;
gni_ct_get_post_descriptor_t *more_get = NULL;
gni_mem_handle_t mdh;
struct gnix_fab_req *more_req;
struct gnix_fid_mem_desc *loc_md;
struct slist_entry *item;
int head_off, head_len = 0, tail_len = 0, entries = 0, idx = 0;
int indirect = !!(req->flags & GNIX_RMA_INDIRECT);
int sub_indirect;
GNIX_DEBUG(FI_LOG_EP_DATA, "FI_MORE: Filling PD\n");
/* If top level fab_req is unaligned, increment entries. */
if (req->type == GNIX_FAB_RQ_RDMA_READ) {
head_off = req->rma.rem_addr & GNI_READ_ALIGN_MASK;
head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
tail_len = (req->rma.rem_addr + req->rma.len)
& GNI_READ_ALIGN_MASK;
if (!indirect) {
if (head_off) {
entries++;
GNIX_DEBUG(FI_LOG_EP_DATA,
"FI_MORE: Txd Head Off\n");
}
if (tail_len) {
entries++;
GNIX_DEBUG(FI_LOG_EP_DATA,
"FI_MORE: Txd Tail Off\n");
}
}
/* Populate txd */
if (indirect) {
GNIX_DEBUG(FI_LOG_EP_DATA, "FI_MORE: Txd Indirect\n");
__gnix_rma_fill_pd_indirect_get(req, txd);
} else {
txd->gni_desc.local_addr = (uint64_t)req->rma.loc_addr + head_len;
txd->gni_desc.remote_addr = (uint64_t)req->rma.rem_addr + head_len;
txd->gni_desc.length = req->rma.len - head_len - tail_len;
assert(txd->gni_desc.length);
}
}
/* Count number of sub post-descriptors to be chained. */
slist_entry_foreach(req->rma.sle.next, item) {
entries++; /* Increment regardless of type */
/* Get fab_req pointer */
if (req->type == GNIX_FAB_RQ_RDMA_READ) {
more_req = container_of(item, struct gnix_fab_req,
rma.sle);
sub_indirect = !!(more_req->flags & GNIX_RMA_INDIRECT);
head_off = more_req->rma.rem_addr & GNI_READ_ALIGN_MASK;
head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
tail_len = (more_req->rma.rem_addr + more_req->rma.len)
& GNI_READ_ALIGN_MASK;
/* Additional increments if unaligned */
if (!sub_indirect) {
if (head_off) {
entries++;
GNIX_DEBUG(FI_LOG_EP_DATA,
"FI_MORE: Chain Req Head off++\n");
}
if (tail_len) {
entries++;
GNIX_DEBUG(FI_LOG_EP_DATA,
"FI_MORE: Chaing Req Tail off++\n");
}
}
}
}
GNIX_INFO(FI_LOG_EP_DATA, "FI_MORE: %d sub descs to be populated\n",
entries);
/* Allocate space for sub descriptors */
if (entries > 0) {
if (req->type == GNIX_FAB_RQ_RDMA_WRITE) {
txd->gni_more_ct_descs = malloc(entries *
sizeof(gni_ct_put_post_descriptor_t));
more_put = (gni_ct_put_post_descriptor_t *)
txd->gni_more_ct_descs;
} else {
txd->gni_more_ct_descs = malloc(entries *
sizeof(gni_ct_get_post_descriptor_t));
more_get = (gni_ct_get_post_descriptor_t *)
txd->gni_more_ct_descs;
/* Populate Head/Tail of TOP Req if req is unaligned */
__gnix_rma_more_fill_sub_htd(req, more_get,
&txd->gni_desc.remote_mem_hndl, &idx, &entries);
}
}
/* Populate sub descriptors */
slist_entry_foreach(req->rma.sle.next, item) {
/* Get fab_req pointer */
more_req = container_of(item, struct gnix_fab_req, rma.sle);
sub_indirect = !!(more_req->flags & GNIX_RMA_INDIRECT);
/* Populate based on type */
if (req->type == GNIX_FAB_RQ_RDMA_WRITE) {
assert(more_put);
more_put[idx].ep_hndl = more_req->vc->gni_ep;
more_put[idx].length = more_req->rma.len;
more_put[idx].remote_addr = more_req->rma.rem_addr;
more_put[idx].local_addr = (uint64_t)more_req->
rma.loc_addr;
_GNIX_CONVERT_MR_KEY(more_req->vc->ep->auth_key->using_vmdh,
more_req->vc->peer_key_offset,
_gnix_convert_key_to_mhdl_no_crc,
&more_req->rma.rem_mr_key, &mdh);
more_put[idx].remote_mem_hndl = mdh;
if (idx < entries - 1)
more_put[idx].next_descr = &more_put[idx + 1];
else
more_put[idx].next_descr = NULL;
idx++;
} else {
assert(more_get);
_GNIX_CONVERT_MR_KEY(more_req->vc->ep->auth_key->using_vmdh,
more_req->vc->peer_key_offset,
_gnix_convert_key_to_mhdl_no_crc,
&more_req->rma.rem_mr_key, &mdh);
more_get[idx].remote_mem_hndl = mdh;
more_get[idx].ep_hndl = more_req->vc->gni_ep;
if (sub_indirect) {
__gnix_rma_more_fill_pd_indirect_get(more_req, more_get, idx);
if (idx < entries - 1) {
more_get[idx].next_descr = &more_get[idx + 1];
} else {
more_get[idx].next_descr = NULL;
}
idx++;
} else {
/* get main aligned data in first transaction */
loc_md = (struct gnix_fid_mem_desc *)more_req->rma.loc_md;
more_get[idx].local_mem_hndl = loc_md->mem_hndl;
more_get[idx].local_addr = (uint64_t)more_req->rma.loc_addr + head_len;
more_get[idx].remote_addr = (uint64_t)more_req->rma.rem_addr + head_len;
more_get[idx].length = more_req->rma.len - head_len - tail_len;
GNIX_DEBUG(FI_LOG_EP_DATA, "rma.len: %d - head_len:%d - tail_len:%d\n",
more_req->rma.len, head_len, tail_len);
assert(more_get[idx].length);
if (idx < entries - 1) {
more_get[idx].next_descr = &more_get[idx + 1];
} else {
more_get[idx].next_descr = NULL;
}
idx++;
/* head/tail function */
__gnix_rma_more_fill_sub_htd(more_req, more_get, &mdh,
&idx, &entries);
}
}
}
if (entries > 0) {
if (req->type == GNIX_FAB_RQ_RDMA_WRITE) {
txd->gni_desc.next_descr = &more_put[0];
} else {
txd->gni_desc.next_descr = &more_get[0];
}
}
}