unsigned int OpenSSLCryptoKeyRSA::privateDecrypt()

in xsec/enc/OpenSSL/OpenSSLCryptoKeyRSA.cpp [658:786]
105 lines of code
15 McCabe index (conditional complexity)

unsigned int OpenSSLCryptoKeyRSA::privateDecrypt(
        const unsigned char* inBuf,
        unsigned char* plainBuf,
        unsigned int inLength,
        unsigned int maxOutLength,
        PaddingType padding,
        const XMLCh* hashURI,
        const XMLCh* mgfURI,
        unsigned char* params,
        unsigned int paramslen) const {

    // Perform a decrypt
    if (mp_rsaKey == NULL) {
        throw XSECCryptoException(XSECCryptoException::RSAError,
            "OpenSSL:RSA - Attempt to decrypt data with empty key");
    }

#if 0
    /* normally commented out code to determine endian problems */
    unsigned int i;
    unsigned char e[2048];
    unsigned char * inBuf1 = (unsigned char *) inBuf;
    if (inLength < 2048) {
        memcpy(e, inBuf, inLength);
        for (i = 0; i < inLength;++i) {
            inBuf1[i] = e[inLength - 1 - i];
        }
    }
#endif

    int decryptSize;

    switch (padding) {

    case XSECCryptoKeyRSA::PAD_PKCS_1_5 :

        decryptSize = RSA_private_decrypt(inLength,
#if defined(XSEC_OPENSSL_CONST_BUFFERS)
                            inBuf,
#else
                            (unsigned char *) inBuf,
#endif
                            plainBuf,
                            mp_rsaKey,
                            RSA_PKCS1_PADDING);

        if (decryptSize < 0) {
            throw XSECCryptoException(XSECCryptoException::RSAError,
                "OpenSSL:RSA privateKeyDecrypt - Error Decrypting PKCS1_5 padded RSA encrypt");
        }

        break;

    case XSECCryptoKeyRSA::PAD_OAEP :
        {
            const EVP_MD* evp_md = getDigestFromHashType(XSECAlgorithmSupport::getHashType(hashURI));
            if (evp_md == NULL) {
                throw XSECCryptoException(XSECCryptoException::UnsupportedAlgorithm,
                    "OpenSSL:RSA - OAEP digest algorithm not supported");
            }


            const EVP_MD* mgf_md = getDigestFromHashType(XSECAlgorithmSupport::getMGF1HashType(mgfURI));
            if (mgf_md == NULL) {
                throw XSECCryptoException(XSECCryptoException::UnsupportedAlgorithm,
                    "OpenSSL:RSA - OAEP MGF algorithm not supported");
            }

            unsigned char * tBuf;
            int num = RSA_size(mp_rsaKey);
            XSECnew(tBuf, unsigned char[num]);
            ArrayJanitor<unsigned char> j_tBuf(tBuf);

            decryptSize = RSA_private_decrypt(inLength,
#if defined(XSEC_OPENSSL_CONST_BUFFERS)
                                inBuf,
#else
                                (unsigned char *) inBuf,
#endif
                                tBuf,
                                mp_rsaKey,
                                RSA_NO_PADDING);
            if (decryptSize < 0) {
                throw XSECCryptoException(XSECCryptoException::RSAError,
                    "OpenSSL:RSA privateKeyDecrypt - Error doing raw decrypt of RSA encrypted data");
            }

            // Clear out the "0"s at the front
            int i;
            for (i = 0; i < num && tBuf[i] == 0; ++i)
                --decryptSize;

            decryptSize = RSA_padding_check_PKCS1_OAEP(plainBuf,
                                                       maxOutLength,
                                                       &tBuf[i],
                                                       decryptSize,
                                                       num,
                                                       params,
                                                       paramslen,
                                                       evp_md,
                                                       mgf_md);

            if (decryptSize < 0) {
                throw XSECCryptoException(XSECCryptoException::RSAError,
                    "OpenSSL:RSA privateKeyDecrypt - Error removing OAEPadding");
            }

        }
        break;

    default :
        throw XSECCryptoException(XSECCryptoException::RSAError,
            "OpenSSL:RSA - Unknown padding method");
    }

#if 0
    /* normally commented out code to determine endian problems */
    int i;
    unsigned char t[512];
    if (decryptSize < 512) {
        memcpy(t, plainBuf, decryptSize);
        for (i = 0; i < decryptSize;++i) {
            plainBuf[i] = t[decryptSize - 1 - i];
        }
    }
#endif

    return decryptSize;
}