absl::Status EncryptRsaOaep()

in kmsp11/util/crypto_utils.cc [288:354]
58 lines of code
13 McCabe index (conditional complexity)

absl::Status EncryptRsaOaep(EVP_PKEY* key, const EVP_MD* hash,
                            absl::Span<const uint8_t> plaintext,
                            absl::Span<uint8_t> ciphertext) {
  if (!key) {
    return NewInvalidArgumentError("missing required argument: key",
                                   CKR_DEVICE_ERROR, SOURCE_LOCATION);
  }
  if (!hash) {
    return NewInvalidArgumentError("missing required argument: hash",
                                   CKR_DEVICE_ERROR, SOURCE_LOCATION);
  }

  const RSA* rsa_key = EVP_PKEY_get0_RSA(key);
  if (!rsa_key) {
    return NewInvalidArgumentError(
        absl::StrFormat("unexpected key type %d provided to EncryptRsaOaep",
                        EVP_PKEY_id(key)),
        CKR_DEVICE_ERROR, SOURCE_LOCATION);
  }

  size_t modulus_size = RSA_size(rsa_key);
  if (ciphertext.size() != modulus_size) {
    return NewInvalidArgumentError(
        absl::StrFormat("unexpected ciphertext size (got %d, want %d)",
                        ciphertext.size(), modulus_size),
        CKR_DEVICE_ERROR, SOURCE_LOCATION);
  }

  // Size limit from https://tools.ietf.org/html/rfc8017#section-7.1.1
  size_t max_plaintext_size = modulus_size - (2 * EVP_MD_size(hash)) - 2;
  if (plaintext.size() > max_plaintext_size) {
    return NewInvalidArgumentError(
        absl::StrFormat("plaintext size %d exceeds maximum %d",
                        plaintext.size(), max_plaintext_size),
        CKR_DATA_LEN_RANGE, SOURCE_LOCATION);
  }

  bssl::UniquePtr<EVP_PKEY_CTX> ctx(EVP_PKEY_CTX_new(key, nullptr));

  if (!ctx || EVP_PKEY_encrypt_init(ctx.get()) != 1 ||
      EVP_PKEY_CTX_set_rsa_padding(ctx.get(), RSA_PKCS1_OAEP_PADDING) != 1 ||
      EVP_PKEY_CTX_set_rsa_mgf1_md(ctx.get(), hash) != 1 ||
      EVP_PKEY_CTX_set_rsa_oaep_md(ctx.get(), hash) != 1) {
    return NewInternalError(
        absl::StrCat("error building encryption context: ", SslErrorToString()),
        SOURCE_LOCATION);
  }

  size_t out_len = ciphertext.size();
  if (EVP_PKEY_encrypt(ctx.get(), ciphertext.data(), &out_len, plaintext.data(),
                       plaintext.size()) != 1) {
    return NewInternalError(
        absl::StrCat("failed to encrypt: ", SslErrorToString()),
        SOURCE_LOCATION);
  }

  if (out_len != modulus_size) {
    std::fill(ciphertext.begin(), ciphertext.end(), 0);
    return NewInternalError(
        absl::StrFormat(
            "actual encrypted length mismatches expected (got %d, expected %d)",
            out_len, modulus_size),
        SOURCE_LOCATION);
  }

  return absl::OkStatus();
}