in sw/siw/siw_qp_tx.c [431:668]
static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s)
{
struct siw_wqe *wqe = &c_tx->wqe_active;
struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx];
struct kvec iov[MAX_ARRAY];
struct page *page_array[MAX_ARRAY];
struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };
int seg = 0, do_crc = c_tx->do_crc, is_kva = 0, rv;
unsigned int data_len = c_tx->bytes_unsent, hdr_len = 0, trl_len = 0,
sge_off = c_tx->sge_off, sge_idx = c_tx->sge_idx,
pbl_idx = c_tx->pbl_idx;
unsigned long kmap_mask = 0L;
if (c_tx->state == SIW_SEND_HDR) {
if (c_tx->use_sendpage) {
rv = siw_tx_ctrl(c_tx, s, MSG_DONTWAIT | MSG_MORE);
if (rv)
goto done;
c_tx->state = SIW_SEND_DATA;
} else {
iov[0].iov_base =
(char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent;
iov[0].iov_len = hdr_len =
c_tx->ctrl_len - c_tx->ctrl_sent;
seg = 1;
}
}
wqe->processed += data_len;
while (data_len) { /* walk the list of SGE's */
unsigned int sge_len = min(sge->length - sge_off, data_len);
unsigned int fp_off = (sge->laddr + sge_off) & ~PAGE_MASK;
struct siw_mem *mem;
if (!(tx_flags(wqe) & SIW_WQE_INLINE)) {
mem = wqe->mem[sge_idx];
is_kva = mem->mem_obj == NULL ? 1 : 0;
} else {
is_kva = 1;
}
if (is_kva && !c_tx->use_sendpage) {
/*
* tx from kernel virtual address: either inline data
* or memory region with assigned kernel buffer
*/
iov[seg].iov_base =
(void *)(uintptr_t)(sge->laddr + sge_off);
iov[seg].iov_len = sge_len;
if (do_crc)
crypto_shash_update(c_tx->mpa_crc_hd,
iov[seg].iov_base,
sge_len);
sge_off += sge_len;
data_len -= sge_len;
seg++;
goto sge_done;
}
while (sge_len) {
size_t plen = min((int)PAGE_SIZE - fp_off, sge_len);
void *kaddr;
if (!is_kva) {
struct page *p;
if (mem->is_pbl)
p = siw_get_pblpage(
mem, sge->laddr + sge_off,
&pbl_idx);
else
p = siw_get_upage(mem->umem,
sge->laddr + sge_off);
if (unlikely(!p)) {
siw_unmap_pages(iov, kmap_mask, seg);
wqe->processed -= c_tx->bytes_unsent;
rv = -EFAULT;
goto done_crc;
}
page_array[seg] = p;
if (!c_tx->use_sendpage) {
void *kaddr = kmap_local_page(p);
/* Remember for later kunmap() */
kmap_mask |= BIT(seg);
iov[seg].iov_base = kaddr + fp_off;
iov[seg].iov_len = plen;
if (do_crc)
crypto_shash_update(
c_tx->mpa_crc_hd,
iov[seg].iov_base,
plen);
} else if (do_crc) {
kaddr = kmap_local_page(p);
crypto_shash_update(c_tx->mpa_crc_hd,
kaddr + fp_off,
plen);
kunmap_local(kaddr);
}
} else {
u64 va = sge->laddr + sge_off;
page_array[seg] = virt_to_page(va & PAGE_MASK);
if (do_crc)
crypto_shash_update(
c_tx->mpa_crc_hd,
(void *)(uintptr_t)va,
plen);
}
sge_len -= plen;
sge_off += plen;
data_len -= plen;
fp_off = 0;
if (++seg > (int)MAX_ARRAY) {
siw_dbg_qp(tx_qp(c_tx), "to many fragments\n");
siw_unmap_pages(iov, kmap_mask, seg-1);
wqe->processed -= c_tx->bytes_unsent;
rv = -EMSGSIZE;
goto done_crc;
}
}
sge_done:
/* Update SGE variables at end of SGE */
if (sge_off == sge->length &&
(data_len != 0 || wqe->processed < wqe->bytes)) {
sge_idx++;
sge++;
sge_off = 0;
}
}
/* trailer */
if (likely(c_tx->state != SIW_SEND_TRAILER)) {
iov[seg].iov_base = &c_tx->trailer.pad[4 - c_tx->pad];
iov[seg].iov_len = trl_len = MAX_TRAILER - (4 - c_tx->pad);
} else {
iov[seg].iov_base = &c_tx->trailer.pad[c_tx->ctrl_sent];
iov[seg].iov_len = trl_len = MAX_TRAILER - c_tx->ctrl_sent;
}
if (c_tx->pad) {
*(u32 *)c_tx->trailer.pad = 0;
if (do_crc)
crypto_shash_update(c_tx->mpa_crc_hd,
(u8 *)&c_tx->trailer.crc - c_tx->pad,
c_tx->pad);
}
if (!c_tx->mpa_crc_hd)
c_tx->trailer.crc = 0;
else if (do_crc)
crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)&c_tx->trailer.crc);
data_len = c_tx->bytes_unsent;
if (c_tx->use_sendpage) {
rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx],
c_tx->sge_off, data_len);
if (rv == data_len) {
rv = kernel_sendmsg(s, &msg, &iov[seg], 1, trl_len);
if (rv > 0)
rv += data_len;
else
rv = data_len;
}
} else {
rv = kernel_sendmsg(s, &msg, iov, seg + 1,
hdr_len + data_len + trl_len);
siw_unmap_pages(iov, kmap_mask, seg);
}
if (rv < (int)hdr_len) {
/* Not even complete hdr pushed or negative rv */
wqe->processed -= data_len;
if (rv >= 0) {
c_tx->ctrl_sent += rv;
rv = -EAGAIN;
}
goto done_crc;
}
rv -= hdr_len;
if (rv >= (int)data_len) {
/* all user data pushed to TCP or no data to push */
if (data_len > 0 && wqe->processed < wqe->bytes) {
/* Save the current state for next tx */
c_tx->sge_idx = sge_idx;
c_tx->sge_off = sge_off;
c_tx->pbl_idx = pbl_idx;
}
rv -= data_len;
if (rv == trl_len) /* all pushed */
rv = 0;
else {
c_tx->state = SIW_SEND_TRAILER;
c_tx->ctrl_len = MAX_TRAILER;
c_tx->ctrl_sent = rv + 4 - c_tx->pad;
c_tx->bytes_unsent = 0;
rv = -EAGAIN;
}
} else if (data_len > 0) {
/* Maybe some user data pushed to TCP */
c_tx->state = SIW_SEND_DATA;
wqe->processed -= data_len - rv;
if (rv) {
/*
* Some bytes out. Recompute tx state based
* on old state and bytes pushed
*/
unsigned int sge_unsent;
c_tx->bytes_unsent -= rv;
sge = &wqe->sqe.sge[c_tx->sge_idx];
sge_unsent = sge->length - c_tx->sge_off;
while (sge_unsent <= rv) {
rv -= sge_unsent;
c_tx->sge_idx++;
c_tx->sge_off = 0;
sge++;
sge_unsent = sge->length;
}
c_tx->sge_off += rv;
}
rv = -EAGAIN;
}
done_crc:
c_tx->do_crc = 0;
done:
return rv;
}