in inside-secure/safexcel_cipher.c [668:895]
static int safexcel_send_req(struct crypto_async_request *base, int ring,
struct safexcel_cipher_req *sreq,
struct scatterlist *src, struct scatterlist *dst,
unsigned int cryptlen, unsigned int assoclen,
unsigned int digestsize, u8 *iv, int *commands,
int *results)
{
struct skcipher_request *areq = skcipher_request_cast(base);
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct safexcel_crypto_priv *priv = ctx->base.priv;
struct safexcel_command_desc *cdesc;
struct safexcel_command_desc *first_cdesc = NULL;
struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
struct scatterlist *sg;
unsigned int totlen;
unsigned int totlen_src = cryptlen + assoclen;
unsigned int totlen_dst = totlen_src;
struct safexcel_token *atoken;
int n_cdesc = 0, n_rdesc = 0;
int queued, i, ret = 0;
bool first = true;
sreq->nr_src = sg_nents_for_len(src, totlen_src);
if (ctx->aead) {
/*
* AEAD has auth tag appended to output for encrypt and
* removed from the output for decrypt!
*/
if (sreq->direction == SAFEXCEL_DECRYPT)
totlen_dst -= digestsize;
else
totlen_dst += digestsize;
memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
&ctx->base.ipad, ctx->state_sz);
if (!ctx->xcm)
memcpy(ctx->base.ctxr->data + (ctx->key_len +
ctx->state_sz) / sizeof(u32), &ctx->base.opad,
ctx->state_sz);
} else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
(sreq->direction == SAFEXCEL_DECRYPT)) {
/*
* Save IV from last crypto input word for CBC modes in decrypt
* direction. Need to do this first in case of inplace operation
* as it will be overwritten.
*/
sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv,
crypto_skcipher_ivsize(skcipher),
(totlen_src -
crypto_skcipher_ivsize(skcipher)));
}
sreq->nr_dst = sg_nents_for_len(dst, totlen_dst);
/*
* Remember actual input length, source buffer length may be
* updated in case of inline operation below.
*/
totlen = totlen_src;
queued = totlen_src;
if (src == dst) {
sreq->nr_src = max(sreq->nr_src, sreq->nr_dst);
sreq->nr_dst = sreq->nr_src;
if (unlikely((totlen_src || totlen_dst) &&
(sreq->nr_src <= 0))) {
dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!",
max(totlen_src, totlen_dst));
return -EINVAL;
}
dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
} else {
if (unlikely(totlen_src && (sreq->nr_src <= 0))) {
dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!",
totlen_src);
return -EINVAL;
}
dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) {
dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!",
totlen_dst);
dma_unmap_sg(priv->dev, src, sreq->nr_src,
DMA_TO_DEVICE);
return -EINVAL;
}
dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
}
memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
if (!totlen) {
/*
* The EIP97 cannot deal with zero length input packets!
* So stuff a dummy command descriptor indicating a 1 byte
* (dummy) input packet, using the context record as source.
*/
first_cdesc = safexcel_add_cdesc(priv, ring,
1, 1, ctx->base.ctxr_dma,
1, 1, ctx->base.ctxr_dma,
&atoken);
if (IS_ERR(first_cdesc)) {
/* No space left in the command descriptor ring */
ret = PTR_ERR(first_cdesc);
goto cdesc_rollback;
}
n_cdesc = 1;
goto skip_cdesc;
}
/* command descriptors */
for_each_sg(src, sg, sreq->nr_src, i) {
int len = sg_dma_len(sg);
/* Do not overflow the request */
if (queued < len)
len = queued;
cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
!(queued - len),
sg_dma_address(sg), len, totlen,
ctx->base.ctxr_dma, &atoken);
if (IS_ERR(cdesc)) {
/* No space left in the command descriptor ring */
ret = PTR_ERR(cdesc);
goto cdesc_rollback;
}
if (!n_cdesc)
first_cdesc = cdesc;
n_cdesc++;
queued -= len;
if (!queued)
break;
}
skip_cdesc:
/* Add context control words and token to first command descriptor */
safexcel_context_control(ctx, base, sreq, first_cdesc);
if (ctx->aead)
safexcel_aead_token(ctx, iv, first_cdesc, atoken,
sreq->direction, cryptlen,
assoclen, digestsize);
else
safexcel_skcipher_token(ctx, iv, first_cdesc, atoken,
cryptlen);
/* result descriptors */
for_each_sg(dst, sg, sreq->nr_dst, i) {
bool last = (i == sreq->nr_dst - 1);
u32 len = sg_dma_len(sg);
/* only allow the part of the buffer we know we need */
if (len > totlen_dst)
len = totlen_dst;
if (unlikely(!len))
break;
totlen_dst -= len;
/* skip over AAD space in buffer - not written */
if (assoclen) {
if (assoclen >= len) {
assoclen -= len;
continue;
}
rdesc = safexcel_add_rdesc(priv, ring, first, last,
sg_dma_address(sg) +
assoclen,
len - assoclen);
assoclen = 0;
} else {
rdesc = safexcel_add_rdesc(priv, ring, first, last,
sg_dma_address(sg),
len);
}
if (IS_ERR(rdesc)) {
/* No space left in the result descriptor ring */
ret = PTR_ERR(rdesc);
goto rdesc_rollback;
}
if (first) {
first_rdesc = rdesc;
first = false;
}
n_rdesc++;
}
if (unlikely(first)) {
/*
* Special case: AEAD decrypt with only AAD data.
* In this case there is NO output data from the engine,
* but the engine still needs a result descriptor!
* Create a dummy one just for catching the result token.
*/
rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
if (IS_ERR(rdesc)) {
/* No space left in the result descriptor ring */
ret = PTR_ERR(rdesc);
goto rdesc_rollback;
}
first_rdesc = rdesc;
n_rdesc = 1;
}
safexcel_rdr_req_set(priv, ring, first_rdesc, base);
*commands = n_cdesc;
*results = n_rdesc;
return 0;
rdesc_rollback:
for (i = 0; i < n_rdesc; i++)
safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
cdesc_rollback:
for (i = 0; i < n_cdesc; i++)
safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
if (src == dst) {
dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL);
} else {
dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE);
}
return ret;
}