ERL_NIF_TERM pkey_sign_nif()

in lib/crypto/c_src/pkey.c [511:745]

• 205 lines of code
• 56 McCabe index (conditional complexity)

ERL_NIF_TERM pkey_sign_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
{/* (Algorithm, Type, Data|{digest,Digest}, Key|#{}, Options) */
    int i;
    int sig_bin_alloc = 0;
    ERL_NIF_TERM ret;
    const EVP_MD *md = NULL;
    unsigned char md_value[EVP_MAX_MD_SIZE];
    EVP_PKEY *pkey = NULL;
#ifdef HAVE_EDDSA
    EVP_MD_CTX *mdctx = NULL;
#endif
#ifdef HAS_EVP_PKEY_CTX
    EVP_PKEY_CTX *ctx = NULL;
    size_t siglen;
#else
    int len;
    unsigned int siglen;
#endif
    PKeySignOptions sig_opt;
    ErlNifBinary sig_bin; /* signature */
    unsigned char *tbs; /* data to be signed */
    size_t tbslen;
    RSA *rsa = NULL;
#ifdef HAVE_DSA
    DSA *dsa = NULL;
#endif
#if defined(HAVE_EC)
    EC_KEY *ec = NULL;
#endif
/*char buf[1024];
enif_get_atom(env,argv[0],buf,1024,ERL_NIF_LATIN1); printf("algo=%s ",buf);
enif_get_atom(env,argv[1],buf,1024,ERL_NIF_LATIN1); printf("hash=%s ",buf);
*/

#ifndef HAS_ENGINE_SUPPORT
    if (enif_is_map(env, argv[3]))
        return atom_notsup;
#endif

    i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen);
    switch (i) {
    case PKEY_OK:
        break;
    case PKEY_NOTSUP:
        goto notsup;
    default:
        goto bad_arg;
    }

    i = get_pkey_sign_options(env, argv[0], argv[4], md, &sig_opt);
    switch (i) {
    case PKEY_OK:
        break;
    case PKEY_NOTSUP:
        goto notsup;
    default:
        goto bad_arg;
    }

    if (get_pkey_private_key(env, argv[0], argv[3], &pkey) != PKEY_OK)
        goto bad_arg;

#ifdef HAS_EVP_PKEY_CTX
    if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL)
        goto err;

    if (argv[0] != atom_eddsa) {
        if (EVP_PKEY_sign_init(ctx) != 1)
            goto err;
        if (md != NULL) {
            if (EVP_PKEY_CTX_set_signature_md(ctx, md) != 1)
                goto err;
        }
    }

    if (argv[0] == atom_rsa) {
        if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) != 1)
            goto err;
# ifdef HAVE_RSA_PKCS1_PSS_PADDING
	if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) {
            if (sig_opt.rsa_mgf1_md != NULL) {
# ifdef HAVE_RSA_MGF1_MD
                if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) != 1)
                    goto err;
# else
                goto notsup;
# endif
            }
            if (sig_opt.rsa_pss_saltlen > -2) {
                if (EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) != 1)
                    goto err;
            }
        }
#endif
    }

    if (argv[0] == atom_eddsa) {
#ifdef HAVE_EDDSA
        if (!FIPS_mode()) {
            if ((mdctx = EVP_MD_CTX_new()) == NULL)
                goto err;

            if (EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey) != 1)
                goto err;
            if (EVP_DigestSign(mdctx, NULL, &siglen, tbs, tbslen) != 1)
                goto err;
            if (!enif_alloc_binary(siglen, &sig_bin))
                goto err;
            sig_bin_alloc = 1;

            if (EVP_DigestSign(mdctx, sig_bin.data, &siglen, tbs, tbslen) != 1)
                goto bad_key;
        }
        else
#endif
            goto notsup;
    } else {
        if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) != 1)
            goto err;
        if (!enif_alloc_binary(siglen, &sig_bin))
            goto err;
        sig_bin_alloc = 1;

        if (md != NULL) {
            ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md));
        }
        if (EVP_PKEY_sign(ctx, sig_bin.data, &siglen, tbs, tbslen) != 1)
            goto bad_key;
    }
#else
/*printf("Old interface\r\n");
 */
    if (argv[0] == atom_rsa) {
        if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL)
            goto err;
        if ((len = RSA_size(rsa)) < 0)
            goto err;
        if (!enif_alloc_binary((size_t)len, &sig_bin))
            goto err;
        sig_bin_alloc = 1;

        if ((len = EVP_MD_size(md)) < 0)
            goto err;
        ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len);

        if (RSA_sign(md->type, tbs, (unsigned int)len, sig_bin.data, &siglen, rsa) != 1)
            goto bad_key;
    } else if (argv[0] == atom_dss) {
        if ((dsa = EVP_PKEY_get1_DSA(pkey)) == NULL)
            goto err;
        if ((len = DSA_size(dsa)) < 0)
            goto err;
        if (!enif_alloc_binary((size_t)len, &sig_bin))
            goto err;
        sig_bin_alloc = 1;

        if ((len = EVP_MD_size(md)) < 0)
            goto err;
        ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len);

        if (DSA_sign(md->type, tbs, len, sig_bin.data, &siglen, dsa) != 1)
            goto bad_key;
    } else if (argv[0] == atom_ecdsa) {
#if defined(HAVE_EC)
        if ((ec = EVP_PKEY_get1_EC_KEY(pkey)) == NULL)
            goto err;
        if ((len = ECDSA_size(ec)) < 0)
            goto err;
        if (!enif_alloc_binary((size_t)len, &sig_bin))
            goto err;
        sig_bin_alloc = 1;

        len = EVP_MD_size(md);
        ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len);

        if (ECDSA_sign(md->type, tbs, len, sig_bin.data, &siglen, ec) != 1)
            goto bad_key;
#else
        goto notsup;
#endif
    } else {
        goto bad_arg;
    }
#endif

    ERL_VALGRIND_MAKE_MEM_DEFINED(sig_bin.data, siglen);
    if (siglen != sig_bin.size) {
        if (!enif_realloc_binary(&sig_bin, siglen))
            goto err;
        ERL_VALGRIND_ASSERT_MEM_DEFINED(sig_bin.data, siglen);
    }
    ret = enif_make_binary(env, &sig_bin);
    sig_bin_alloc = 0;
    goto done;

 bad_key:
    ret = atom_error;
    goto done;

 notsup:
    ret = atom_notsup;
    goto done;

 bad_arg:
 err:
    ret = enif_make_badarg(env);
    goto done;

 done:
    if (sig_bin_alloc)
        enif_release_binary(&sig_bin);
    if (rsa)
        RSA_free(rsa);
#ifdef HAVE_DSA
    if (dsa)
        DSA_free(dsa);
#endif
#ifdef HAVE_EC
    if (ec)
        EC_KEY_free(ec);
#endif
#ifdef HAS_EVP_PKEY_CTX
    if (ctx)
        EVP_PKEY_CTX_free(ctx);
#endif
    if (pkey)
        EVP_PKEY_free(pkey);

#ifdef HAVE_EDDSA
    if (mdctx)
        EVP_MD_CTX_free(mdctx);
#endif

    return ret;
}