static int create_openssl_instance()

in adapters/tlsio_openssl.c [1059:1233]

• 165 lines of code
• 21 McCabe index (conditional complexity)

static int create_openssl_instance(TLS_IO_INSTANCE* tlsInstance)
{
    int result;

    const SSL_METHOD* method = NULL;

#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
    if (tlsInstance->tls_version == VERSION_1_2)
    {
        method = TLSv1_2_method();
    }
    else if (tlsInstance->tls_version == VERSION_1_1)
    {
        method = TLSv1_1_method();
    }
    else
    {
        method = TLSv1_method();
    }
#else
    {
        method = TLS_method();
    }
#endif

    tlsInstance->ssl_context = SSL_CTX_new(method);
    if (tlsInstance->ssl_context == NULL)
    {
        log_ERR_get_error("Failed allocating OpenSSL context.");
        result = MU_FAILURE;
    }
    #ifndef OPENSSL_NO_ENGINE
    else if ((tlsInstance->engine_id != NULL) &&
             (engine_load(tlsInstance) != 0))
    {
        SSL_CTX_free(tlsInstance->ssl_context);
        tlsInstance->ssl_context = NULL;
        result = MU_FAILURE;
    }
    #endif // OPENSSL_NO_ENGINE
    else if ((tlsInstance->cipher_list != NULL) &&
             (SSL_CTX_set_cipher_list(tlsInstance->ssl_context, tlsInstance->cipher_list)) != 1)
    {
        engine_destroy(tlsInstance);
        SSL_CTX_free(tlsInstance->ssl_context);
        tlsInstance->ssl_context = NULL;
        log_ERR_get_error("unable to set cipher list.");
        result = MU_FAILURE;
    }
    else if (add_certificate_to_store(tlsInstance, tlsInstance->certificate) != 0)
    {
        engine_destroy(tlsInstance);
        SSL_CTX_free(tlsInstance->ssl_context);
        tlsInstance->ssl_context = NULL;
        log_ERR_get_error("unable to add_certificate_to_store.");
        result = MU_FAILURE;
    }
    /*x509 authentication can only be build before underlying connection is realized*/
    else if (
        (tlsInstance->x509_certificate != NULL) &&
        (tlsInstance->x509_private_key != NULL) &&
        (x509_openssl_add_credentials(
            tlsInstance->ssl_context, 
            tlsInstance->x509_certificate, 
            tlsInstance->x509_private_key,
    #ifndef OPENSSL_NO_ENGINE
            tlsInstance->x509_private_key_type,
            tlsInstance->engine) != 0)
    #else // OPENSSL_NO_ENGINE
            tlsInstance->x509_private_key_type) != 0)
    #endif // OPENSSL_NO_ENGINE
        )
    {
        engine_destroy(tlsInstance);
        SSL_CTX_free(tlsInstance->ssl_context);
        tlsInstance->ssl_context = NULL;
        log_ERR_get_error("unable to use x509 authentication");
        result = MU_FAILURE;
    }
    else
    {
        SSL_CTX_set_cert_verify_callback(tlsInstance->ssl_context, tlsInstance->tls_validation_callback, tlsInstance->tls_validation_callback_data);

        tlsInstance->in_bio = BIO_new(BIO_s_mem());
        if (tlsInstance->in_bio == NULL)
        {
            engine_destroy(tlsInstance);
            SSL_CTX_free(tlsInstance->ssl_context);
            tlsInstance->ssl_context = NULL;
            log_ERR_get_error("Failed BIO_new for in BIO.");
            result = MU_FAILURE;
        }
        else
        {
            tlsInstance->out_bio = BIO_new(BIO_s_mem());
            if (tlsInstance->out_bio == NULL)
            {
                (void)BIO_free(tlsInstance->in_bio);
                engine_destroy(tlsInstance);
                SSL_CTX_free(tlsInstance->ssl_context);
                tlsInstance->ssl_context = NULL;
                log_ERR_get_error("Failed BIO_new for out BIO.");
                result = MU_FAILURE;
            }
            else
            {
                if ((BIO_set_mem_eof_return(tlsInstance->in_bio, -1) <= 0) ||
                    (BIO_set_mem_eof_return(tlsInstance->out_bio, -1) <= 0))
                {
                    (void)BIO_free(tlsInstance->in_bio);
                    (void)BIO_free(tlsInstance->out_bio);
                    engine_destroy(tlsInstance);
                    SSL_CTX_free(tlsInstance->ssl_context);
                    tlsInstance->ssl_context = NULL;
                    LogError("Failed BIO_set_mem_eof_return.");
                    result = MU_FAILURE;
                }
                else
                {
                    SSL_CTX_set_verify(tlsInstance->ssl_context, SSL_VERIFY_PEER, NULL);

                    // Specifies that the default locations for which CA certificates are loaded should be used.
                    if (SSL_CTX_set_default_verify_paths(tlsInstance->ssl_context) != 1)
                    {
                        // This is only a warning to the user. They can still specify the certificate via SetOption.
                        LogInfo("WARNING: Unable to specify the default location for CA certificates on this platform.");
                    }

                    tlsInstance->ssl = SSL_new(tlsInstance->ssl_context);

                    if (tlsInstance->ssl == NULL)
                    {
                        (void)BIO_free(tlsInstance->in_bio);
                        (void)BIO_free(tlsInstance->out_bio);
                        engine_destroy(tlsInstance);
                        SSL_CTX_free(tlsInstance->ssl_context);
                        tlsInstance->ssl_context = NULL;
                        log_ERR_get_error("Failed creating OpenSSL instance.");
                        result = MU_FAILURE;
                    }
                    else if (SSL_set_tlsext_host_name(tlsInstance->ssl, tlsInstance->hostname) != 1)
                    {
                        SSL_free(tlsInstance->ssl);
                        tlsInstance->ssl = NULL;
                        (void)BIO_free(tlsInstance->in_bio);
                        (void)BIO_free(tlsInstance->out_bio);
                        engine_destroy(tlsInstance);
                        SSL_CTX_free(tlsInstance->ssl_context);
                        tlsInstance->ssl_context = NULL;
                        log_ERR_get_error("Failed setting SNI hostname hint.");
                        result = MU_FAILURE;
                    }
                    else if (enable_domain_check(tlsInstance))
                    {
                        SSL_free(tlsInstance->ssl);
                        tlsInstance->ssl = NULL;
                        (void)BIO_free(tlsInstance->in_bio);
                        (void)BIO_free(tlsInstance->out_bio);
                        SSL_CTX_free(tlsInstance->ssl_context);
                        tlsInstance->ssl_context = NULL;
                        log_ERR_get_error("Failed to configure domain name verification.");
                        result = MU_FAILURE;
                    }
                    else
                    {
                        SSL_set_bio(tlsInstance->ssl, tlsInstance->in_bio, tlsInstance->out_bio);
                        SSL_set_connect_state(tlsInstance->ssl);
                        result = 0;
                    }
                }
            }
        }
    }
    return result;
}