in bcm/cipher.c [653:885]
static int handle_ahash_req(struct iproc_reqctx_s *rctx)
{
struct spu_hw *spu = &iproc_priv.spu;
struct crypto_async_request *areq = rctx->parent;
struct ahash_request *req = ahash_request_cast(areq);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
struct crypto_tfm *tfm = crypto_ahash_tfm(ahash);
unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
struct iproc_ctx_s *ctx = rctx->ctx;
/* number of bytes still to be hashed in this req */
unsigned int nbytes_to_hash = 0;
int err;
unsigned int chunksize = 0; /* length of hash carry + new data */
/*
* length of new data, not from hash carry, to be submitted in
* this hw request
*/
unsigned int new_data_len;
unsigned int __maybe_unused chunk_start = 0;
u32 db_size; /* Length of data field, incl gcm and hash padding */
int pad_len = 0; /* total pad len, including gcm, hash, stat padding */
u32 data_pad_len = 0; /* length of GCM/CCM padding */
u32 stat_pad_len = 0; /* length of padding to align STATUS word */
struct brcm_message *mssg; /* mailbox message */
struct spu_request_opts req_opts;
struct spu_cipher_parms cipher_parms;
struct spu_hash_parms hash_parms;
struct spu_aead_parms aead_parms;
unsigned int local_nbuf;
u32 spu_hdr_len;
unsigned int digestsize;
u16 rem = 0;
/*
* number of entries in src and dst sg. Always includes SPU msg header.
* rx always includes a buffer to catch digest and STATUS.
*/
u8 rx_frag_num = 3;
u8 tx_frag_num = 1;
flow_log("total_todo %u, total_sent %u\n",
rctx->total_todo, rctx->total_sent);
memset(&req_opts, 0, sizeof(req_opts));
memset(&cipher_parms, 0, sizeof(cipher_parms));
memset(&hash_parms, 0, sizeof(hash_parms));
memset(&aead_parms, 0, sizeof(aead_parms));
req_opts.bd_suppress = true;
hash_parms.alg = ctx->auth.alg;
hash_parms.mode = ctx->auth.mode;
hash_parms.type = HASH_TYPE_NONE;
hash_parms.key_buf = (u8 *)ctx->authkey;
hash_parms.key_len = ctx->authkeylen;
/*
* For hash algorithms below assignment looks bit odd but
* it's needed for AES-XCBC and AES-CMAC hash algorithms
* to differentiate between 128, 192, 256 bit key values.
* Based on the key values, hash algorithm is selected.
* For example for 128 bit key, hash algorithm is AES-128.
*/
cipher_parms.type = ctx->cipher_type;
mssg = &rctx->mb_mssg;
chunk_start = rctx->src_sent;
/*
* Compute the amount remaining to hash. This may include data
* carried over from previous requests.
*/
nbytes_to_hash = rctx->total_todo - rctx->total_sent;
chunksize = nbytes_to_hash;
if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
(chunksize > ctx->max_payload))
chunksize = ctx->max_payload;
/*
* If this is not a final request and the request data is not a multiple
* of a full block, then simply park the extra data and prefix it to the
* data for the next request.
*/
if (!rctx->is_final) {
u8 *dest = rctx->hash_carry + rctx->hash_carry_len;
u16 new_len; /* len of data to add to hash carry */
rem = chunksize % blocksize; /* remainder */
if (rem) {
/* chunksize not a multiple of blocksize */
chunksize -= rem;
if (chunksize == 0) {
/* Don't have a full block to submit to hw */
new_len = rem - rctx->hash_carry_len;
sg_copy_part_to_buf(req->src, dest, new_len,
rctx->src_sent);
rctx->hash_carry_len = rem;
flow_log("Exiting with hash carry len: %u\n",
rctx->hash_carry_len);
packet_dump(" buf: ",
rctx->hash_carry,
rctx->hash_carry_len);
return -EAGAIN;
}
}
}
/* if we have hash carry, then prefix it to the data in this request */
local_nbuf = rctx->hash_carry_len;
rctx->hash_carry_len = 0;
if (local_nbuf)
tx_frag_num++;
new_data_len = chunksize - local_nbuf;
/* Count number of sg entries to be used in this request */
rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip,
new_data_len);
/* AES hashing keeps key size in type field, so need to copy it here */
if (hash_parms.alg == HASH_ALG_AES)
hash_parms.type = (enum hash_type)cipher_parms.type;
else
hash_parms.type = spu->spu_hash_type(rctx->total_sent);
digestsize = spu->spu_digest_size(ctx->digestsize, ctx->auth.alg,
hash_parms.type);
hash_parms.digestsize = digestsize;
/* update the indexes */
rctx->total_sent += chunksize;
/* if you sent a prebuf then that wasn't from this req->src */
rctx->src_sent += new_data_len;
if ((rctx->total_sent == rctx->total_todo) && rctx->is_final)
hash_parms.pad_len = spu->spu_hash_pad_len(hash_parms.alg,
hash_parms.mode,
chunksize,
blocksize);
/*
* If a non-first chunk, then include the digest returned from the
* previous chunk so that hw can add to it (except for AES types).
*/
if ((hash_parms.type == HASH_TYPE_UPDT) &&
(hash_parms.alg != HASH_ALG_AES)) {
hash_parms.key_buf = rctx->incr_hash;
hash_parms.key_len = digestsize;
}
atomic64_add(chunksize, &iproc_priv.bytes_out);
flow_log("%s() final: %u nbuf: %u ",
__func__, rctx->is_final, local_nbuf);
if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
flow_log("max_payload infinite\n");
else
flow_log("max_payload %u\n", ctx->max_payload);
flow_log("chunk_start: %u chunk_size: %u\n", chunk_start, chunksize);
/* Prepend SPU header with type 3 BCM header */
memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
hash_parms.prebuf_len = local_nbuf;
spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
BCM_HDR_LEN,
&req_opts, &cipher_parms,
&hash_parms, &aead_parms,
new_data_len);
if (spu_hdr_len == 0) {
pr_err("Failed to create SPU request header\n");
return -EFAULT;
}
/*
* Determine total length of padding required. Put all padding in one
* buffer.
*/
data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, chunksize);
db_size = spu_real_db_size(0, 0, local_nbuf, new_data_len,
0, 0, hash_parms.pad_len);
if (spu->spu_tx_status_len())
stat_pad_len = spu->spu_wordalign_padlen(db_size);
if (stat_pad_len)
rx_frag_num++;
pad_len = hash_parms.pad_len + data_pad_len + stat_pad_len;
if (pad_len) {
tx_frag_num++;
spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len,
hash_parms.pad_len, ctx->auth.alg,
ctx->auth.mode, rctx->total_sent,
stat_pad_len);
}
spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
spu_hdr_len);
packet_dump(" prebuf: ", rctx->hash_carry, local_nbuf);
flow_log("Data:\n");
dump_sg(rctx->src_sg, rctx->src_skip, new_data_len);
packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len);
/*
* Build mailbox message containing SPU request msg and rx buffers
* to catch response message
*/
memset(mssg, 0, sizeof(*mssg));
mssg->type = BRCM_MESSAGE_SPU;
mssg->ctx = rctx; /* Will be returned in response */
/* Create rx scatterlist to catch result */
err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize,
stat_pad_len);
if (err)
return err;
/* Create tx scatterlist containing SPU request message */
tx_frag_num += rctx->src_nents;
if (spu->spu_tx_status_len())
tx_frag_num++;
err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
local_nbuf, new_data_len, pad_len);
if (err)
return err;
err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
if (unlikely(err < 0))
return err;
return -EINPROGRESS;
}