AttestationResult EncryptDataWithRSAPubKey()

in client-library/src/Attestation/AttestationClient/lib/AttestationLibUtils.cpp [455:579]
104 lines of code
18 McCabe index (conditional complexity)

    AttestationResult EncryptDataWithRSAPubKey(BIO* pkey_bio,
                                               const attest::RsaScheme rsaWrapAlgId,
                                               const attest::RsaHashAlg rsaHashAlgId,
                                               const Buffer& input_data,
                                               Buffer& encrypted_data)
    {
        AttestationResult result(AttestationResult::ErrorCode::SUCCESS);
        if (pkey_bio == NULL ||
            input_data.empty()) {
            return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_INVALID_INPUT_PARAMETER,
                                        "Invalid input parameter");
        }

        const EVP_MD* rsa_md = EVP_md_null();
        switch (rsaHashAlgId)
        {
            case RsaHashAlg::RsaSha1:
                rsa_md = EVP_sha1();
                break;
            case RsaHashAlg::RsaSha256:
                rsa_md = EVP_sha256();
                break;
            case RsaHashAlg::RsaSha384:
                rsa_md = EVP_sha384();
                break;
            case RsaHashAlg::RsaSha512:
                rsa_md = EVP_sha512();
                break;
            default:
                return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_EVP_PKEY_ENCRYPT_INIT_FAILED,
                                            "EncryptDataWithRSAPubKey failed; called with unknown message digest algorithm");
        }

        // Set the RSA padding and message digest algorithm
        int ret = 0;
        int rsa_padding_algo = 0;
        switch (rsaWrapAlgId)
        {
            case RsaScheme::RsaEs:
                rsa_padding_algo = RSA_PKCS1_PADDING;
                break;
            case RsaScheme::RsaOaep:
                rsa_padding_algo = RSA_PKCS1_OAEP_PADDING;
                break;
            case RsaScheme::RsaNull:
                rsa_padding_algo = RSA_NO_PADDING;
                break;
            default:
                return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_EVP_PKEY_ENCRYPT_INIT_FAILED,
                                     "EncryptDataWithRSAPubKey failed; called with unknown RSA padding algorithm");
        }

        EVP_PKEY* pkey = PEM_read_bio_PUBKEY(pkey_bio, NULL, NULL, NULL);
        EVP_PKEY_CTX* enc_ctx = EVP_PKEY_CTX_new(pkey, NULL);
        if (EVP_PKEY_encrypt_init(enc_ctx) <= 0) {
            EVP_PKEY_CTX_free(enc_ctx);
            return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_EVP_PKEY_ENCRYPT_INIT_FAILED,
                                        "EVP_PKEY_encrypt_init failed");
        }

        // Set the RSA padding algorithm
        ret = EVP_PKEY_CTX_set_rsa_padding(enc_ctx, rsa_padding_algo);
        if (ret <= 0)
        {
            EVP_PKEY_CTX_free(enc_ctx);
            return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_EVP_PKEY_ENCRYPT_INIT_FAILED,
                "EVP_PKEY_CTX_set_rsa_padding failed");
        }

        // Set the RSA message digest algorithm
        if (rsaWrapAlgId == RsaScheme::RsaOaep)
        {
            ret = EVP_PKEY_CTX_set_rsa_oaep_md(enc_ctx, rsa_md);
            if (ret <= 0)
            {
                EVP_PKEY_CTX_free(enc_ctx);
                return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_EVP_PKEY_ENCRYPT_INIT_FAILED,
                                            "EVP_PKEY_CTX_set_rsa_oaep_md failed");
            }
        }
        else if (rsaWrapAlgId == RsaScheme::RsaEs)
        {
            // TODO: There isn't equivalent of EVP_PKEY_CTX_set_rsa_oaep_md for RSA_PKCS1_PADDING
            //       Need to figure out how to set the hash algorithm for RSA_PKCS1_PADDING
            // Note: 1- EVP_PKEY_CTX_set_rsa_oaep_md is only used for RSA_PKCS1_OAEP_PADDING
            //       and not for RSA_PKCS1_PADDING. EVP_PKEY_CTX_set_rsa_oaep_md throws on Linux,
            //       but not on Windows.
            //       2- EVP_PKEY_CTX_set_signature_md is for signing and not for encryption.

            // ret = EVP_PKEY_CTX_set_rsa_???_md(enc_ctx, rsa_md);
        }
        else if (rsaWrapAlgId == RsaScheme::RsaNull)
        {
            // No need to set any MD for RSA_NO_PADDING
        }
        else
        {
            // Should never get here, since we already checked for valid values
            EVP_PKEY_CTX_free(enc_ctx);
            return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_EVP_PKEY_ENCRYPT_INIT_FAILED,
                                        "Invalid RSA wrap algorithm");
        }

        // Encrypt the data
        size_t outlen;
        unsigned char* out;
        if (EVP_PKEY_encrypt(enc_ctx, NULL, &outlen, &input_data.front(), input_data.size()) <= 0) {
            EVP_PKEY_CTX_free(enc_ctx);
            return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_EVP_PKEY_ENCRYPT_FAILED,
                                        "EVP_PKEY_encrypt failed");
        }
        out = (unsigned char*)OPENSSL_malloc(outlen);
        if (EVP_PKEY_encrypt(enc_ctx, out, &outlen, &input_data.front(), input_data.size()) <= 0) {
            EVP_PKEY_CTX_free(enc_ctx);
            OPENSSL_free(out);
            return LogErrorAndGetResult(AttestationResult::ErrorCode::ERROR_EVP_PKEY_ENCRYPT_FAILED,
                                        "EVP_PKEY_encrypt failed");
        }

        Buffer out_data(out, out + outlen);
        encrypted_data = out_data;
        EVP_PKEY_CTX_free(enc_ctx);
        OPENSSL_free(out);
        return result;
    }