int RSA_padding_check_PKCS1_OAEP()

in xsec/enc/OpenSSL/OpenSSLCryptoKeyRSA.cpp [168:260]

• 70 lines of code
• 17 McCabe index (conditional complexity)

    int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
        const unsigned char *from, int flen, int num,
        const unsigned char *param, int plen,
        const EVP_MD* digest,
        const EVP_MD* mgf_digest)
    {
        int i, digestlen = EVP_MD_size(digest), dblen, mlen = -1;
        const unsigned char *maskeddb;
        int lzero;
        unsigned char *db = NULL, seed[EVP_MAX_MD_SIZE], phash[EVP_MAX_MD_SIZE];
        unsigned char *padded_from;
        int bad = 0;

        if (--num < 2 * digestlen + 1)
            /* 'num' is the length of the modulus, i.e. does not depend on the
             * particular ciphertext. */
            goto decoding_err;

        lzero = num - flen;
        if (lzero < 0)
            {
            /* signalling this error immediately after detection might allow
             * for side-channel attacks (e.g. timing if 'plen' is huge
             * -- cf. James H. Manger, "A Chosen Ciphertext Attack on RSA Optimal
             * Asymmetric Encryption Padding (OAEP) [...]", CRYPTO 2001),
             * so we use a 'bad' flag */
            bad = 1;
            lzero = 0;
            flen = num; /* don't overflow the memcpy to padded_from */
            }

        dblen = num - digestlen;
        db = (unsigned char*) OPENSSL_malloc(dblen + num);
        if (db == NULL)
            {
            RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
            return -1;
            }

        /* Always do this zero-padding copy (even when lzero == 0)
         * to avoid leaking timing info about the value of lzero. */
        padded_from = db + dblen;
        memset(padded_from, 0, lzero);
        memcpy(padded_from + lzero, from, flen);

        maskeddb = padded_from + digestlen;

        if (MGF1(seed, digestlen, maskeddb, dblen, mgf_digest))
            return -1;
        for (i = 0; i < digestlen; i++)
            seed[i] ^= padded_from[i];
  
        if (MGF1(db, dblen, seed, digestlen, mgf_digest))
            return -1;
        for (i = 0; i < dblen; i++)
            db[i] ^= maskeddb[i];

        if (!EVP_Digest((void *)param, plen, phash, NULL, digest, NULL))
            return -1;

        if (memcmp(db, phash, digestlen) != 0 || bad)
            goto decoding_err;
        else
            {
            for (i = digestlen; i < dblen; i++)
                if (db[i] != 0x00)
                    break;
            if (i == dblen || db[i] != 0x01)
                goto decoding_err;
            else
                {
                /* everything looks OK */

                mlen = dblen - ++i;
                if (tlen < mlen)
                    {
                    RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_DATA_TOO_LARGE);
                    mlen = -1;
                    }
                else
                    memcpy(to, db + i, mlen);
                }
            }
        OPENSSL_free(db);
        return mlen;

    decoding_err:
        /* to avoid chosen ciphertext attacks, the error message should not reveal
         * which kind of decoding error happened */
        RSAerr(RSA_F_RSA_PADDING_CHECK_PKCS1_OAEP, RSA_R_OAEP_DECODING_ERROR);
        if (db != NULL) OPENSSL_free(db);
        return -1;
    }