in cdns3/cdns3-gadget.c [1110:1363]
static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
struct usb_request *request)
{
struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
struct cdns3_request *priv_req;
struct cdns3_trb *trb;
struct cdns3_trb *link_trb = NULL;
dma_addr_t trb_dma;
u32 togle_pcs = 1;
int sg_iter = 0;
int num_trb;
int address;
u32 control;
int pcs;
u16 total_tdl = 0;
struct scatterlist *s = NULL;
bool sg_supported = !!(request->num_mapped_sgs);
if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
num_trb = priv_ep->interval;
else
num_trb = sg_supported ? request->num_mapped_sgs : 1;
if (num_trb > priv_ep->free_trbs) {
priv_ep->flags |= EP_RING_FULL;
return -ENOBUFS;
}
priv_req = to_cdns3_request(request);
address = priv_ep->endpoint.desc->bEndpointAddress;
priv_ep->flags |= EP_PENDING_REQUEST;
/* must allocate buffer aligned to 8 */
if (priv_req->flags & REQUEST_UNALIGNED)
trb_dma = priv_req->aligned_buf->dma;
else
trb_dma = request->dma;
trb = priv_ep->trb_pool + priv_ep->enqueue;
priv_req->start_trb = priv_ep->enqueue;
priv_req->trb = trb;
cdns3_select_ep(priv_ep->cdns3_dev, address);
/* prepare ring */
if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
int doorbell, dma_index;
u32 ch_bit = 0;
doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
/* Driver can't update LINK TRB if it is current processed. */
if (doorbell && dma_index == priv_ep->num_trbs - 1) {
priv_ep->flags |= EP_DEFERRED_DRDY;
return -ENOBUFS;
}
/*updating C bt in Link TRB before starting DMA*/
link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
/*
* For TRs size equal 2 enabling TRB_CHAIN for epXin causes
* that DMA stuck at the LINK TRB.
* On the other hand, removing TRB_CHAIN for longer TRs for
* epXout cause that DMA stuck after handling LINK TRB.
* To eliminate this strange behavioral driver set TRB_CHAIN
* bit only for TR size > 2.
*/
if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
TRBS_PER_SEGMENT > 2)
ch_bit = TRB_CHAIN;
link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
}
if (priv_dev->dev_ver <= DEV_VER_V2)
togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
if (sg_supported)
s = request->sg;
/* set incorrect Cycle Bit for first trb*/
control = priv_ep->pcs ? 0 : TRB_CYCLE;
trb->length = 0;
if (priv_dev->dev_ver >= DEV_VER_V2) {
u16 td_size;
td_size = DIV_ROUND_UP(request->length,
priv_ep->endpoint.maxpacket);
if (priv_dev->gadget.speed == USB_SPEED_SUPER)
trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
else
control |= TRB_TDL_HS_SIZE(td_size);
}
do {
u32 length;
/* fill TRB */
control |= TRB_TYPE(TRB_NORMAL);
if (sg_supported) {
trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
length = sg_dma_len(s);
} else {
trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
length = request->length;
}
if (priv_ep->flags & EP_TDLCHK_EN)
total_tdl += DIV_ROUND_UP(length,
priv_ep->endpoint.maxpacket);
trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) |
TRB_LEN(length));
pcs = priv_ep->pcs ? TRB_CYCLE : 0;
/*
* first trb should be prepared as last to avoid processing
* transfer to early
*/
if (sg_iter != 0)
control |= pcs;
if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
control |= TRB_IOC | TRB_ISP;
} else {
/* for last element in TD or in SG list */
if (sg_iter == (num_trb - 1) && sg_iter != 0)
control |= pcs | TRB_IOC | TRB_ISP;
}
if (sg_iter)
trb->control = cpu_to_le32(control);
else
priv_req->trb->control = cpu_to_le32(control);
if (sg_supported) {
trb->control |= cpu_to_le32(TRB_ISP);
/* Don't set chain bit for last TRB */
if (sg_iter < num_trb - 1)
trb->control |= cpu_to_le32(TRB_CHAIN);
s = sg_next(s);
}
control = 0;
++sg_iter;
priv_req->end_trb = priv_ep->enqueue;
cdns3_ep_inc_enq(priv_ep);
trb = priv_ep->trb_pool + priv_ep->enqueue;
trb->length = 0;
} while (sg_iter < num_trb);
trb = priv_req->trb;
priv_req->flags |= REQUEST_PENDING;
priv_req->num_of_trb = num_trb;
if (sg_iter == 1)
trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP);
if (priv_dev->dev_ver < DEV_VER_V2 &&
(priv_ep->flags & EP_TDLCHK_EN)) {
u16 tdl = total_tdl;
u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
if (tdl > EP_CMD_TDL_MAX) {
tdl = EP_CMD_TDL_MAX;
priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
}
if (old_tdl < tdl) {
tdl -= old_tdl;
writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
&priv_dev->regs->ep_cmd);
}
}
/*
* Memory barrier - cycle bit must be set before other filds in trb.
*/
wmb();
/* give the TD to the consumer*/
if (togle_pcs)
trb->control = trb->control ^ cpu_to_le32(1);
if (priv_dev->dev_ver <= DEV_VER_V2)
cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
if (num_trb > 1) {
int i = 0;
while (i < num_trb) {
trace_cdns3_prepare_trb(priv_ep, trb + i);
if (trb + i == link_trb) {
trb = priv_ep->trb_pool;
num_trb = num_trb - i;
i = 0;
} else {
i++;
}
}
} else {
trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
}
/*
* Memory barrier - Cycle Bit must be set before trb->length and
* trb->buffer fields.
*/
wmb();
/*
* For DMULT mode we can set address to transfer ring only once after
* enabling endpoint.
*/
if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
/*
* Until SW is not ready to handle the OUT transfer the ISO OUT
* Endpoint should be disabled (EP_CFG.ENABLE = 0).
* EP_CFG_ENABLE must be set before updating ep_traddr.
*/
if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
!(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
EP_CFG_ENABLE);
}
writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
priv_req->start_trb * TRB_SIZE),
&priv_dev->regs->ep_traddr);
priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
}
if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
trace_cdns3_ring(priv_ep);
/*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
cdns3_rearm_drdy_if_needed(priv_ep);
trace_cdns3_doorbell_epx(priv_ep->name,
readl(&priv_dev->regs->ep_traddr));
}
/* WORKAROUND for transition to L0 */
__cdns3_gadget_wakeup(priv_dev);
return 0;
}