in google/gve/gve_rx_dqo.c [703:796]
int gve_rx_poll_dqo(struct gve_notify_block *block, int budget)
{
struct napi_struct *napi = &block->napi;
netdev_features_t feat = napi->dev->features;
struct gve_rx_ring *rx = block->rx;
struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq;
u32 work_done = 0;
u64 bytes = 0;
int err;
while (work_done < budget) {
struct gve_rx_compl_desc_dqo *compl_desc =
&complq->desc_ring[complq->head];
u32 pkt_bytes;
/* No more new packets */
if (compl_desc->generation == complq->cur_gen_bit)
break;
/* Prefetch the next two descriptors. */
prefetch(&complq->desc_ring[(complq->head + 1) & complq->mask]);
prefetch(&complq->desc_ring[(complq->head + 2) & complq->mask]);
/* Do not read data until we own the descriptor */
dma_rmb();
err = gve_rx_dqo(napi, rx, compl_desc, complq->head, rx->q_num);
if (err < 0) {
gve_rx_free_skb(napi, rx);
u64_stats_update_begin(&rx->statss);
if (err == -ENOMEM)
rx->rx_skb_alloc_fail++;
else if (err == -EINVAL)
rx->rx_desc_err_dropped_pkt++;
u64_stats_update_end(&rx->statss);
}
complq->head = (complq->head + 1) & complq->mask;
complq->num_free_slots++;
/* When the ring wraps, the generation bit is flipped. */
complq->cur_gen_bit ^= (complq->head == 0);
/* Receiving a completion means we have space to post another
* buffer on the buffer queue.
*/
{
struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq;
bufq->head = (bufq->head + 1) & bufq->mask;
}
/* Free running counter of completed descriptors */
rx->cnt++;
if (!rx->ctx.skb_head)
continue;
if (!compl_desc->end_of_packet)
continue;
work_done++;
pkt_bytes = rx->ctx.skb_head->len;
/* The ethernet header (first ETH_HLEN bytes) is snipped off
* by eth_type_trans.
*/
if (skb_headlen(rx->ctx.skb_head))
pkt_bytes += ETH_HLEN;
/* gve_rx_complete_skb() will consume skb if successful */
if (gve_rx_complete_skb(rx, napi, compl_desc, feat) != 0) {
gve_rx_free_skb(napi, rx);
u64_stats_update_begin(&rx->statss);
rx->rx_desc_err_dropped_pkt++;
u64_stats_update_end(&rx->statss);
continue;
}
bytes += pkt_bytes;
rx->ctx.skb_head = NULL;
rx->ctx.skb_tail = NULL;
}
gve_rx_post_buffers_dqo(rx);
u64_stats_update_begin(&rx->statss);
rx->rpackets += work_done;
rx->rbytes += bytes;
u64_stats_update_end(&rx->statss);
return work_done;
}