int SSLconnect::sslHandshake()

in nlsCppSdk/transport/SSLconnect.cpp [81:184]
87 lines of code
14 McCabe index (conditional complexity)

int SSLconnect::sslHandshake(int socketFd, const char *hostname) {
  // LOG_DEBUG("Begin sslHandshake.");
  if (_sslCtx == NULL) {
    LOG_ERROR("SSL(%p) _sslCtx has been released.", this);
    return -(SslCtxEmpty);
  }

  MUTEX_LOCK(_mtxSSL);

  int ret;
  if (_ssl == NULL) {
    _ssl = SSL_new(_sslCtx);
    if (_ssl == NULL) {
      memset(_errorMsg, 0x0, MaxSslErrorLength);
      const char *SSL_new_ret = "return of SSL_new: ";
      const int SSL_new_str_size = strnlen(SSL_new_ret, 24);
      memcpy(_errorMsg, SSL_new_ret, SSL_new_str_size);
      ERR_error_string_n(ERR_get_error(), _errorMsg + SSL_new_str_size,
                         MaxSslErrorLength - SSL_new_str_size - 1);
      LOG_ERROR("SSL(%p) Invoke SSL_new failed:%s.", this, _errorMsg);
      MUTEX_UNLOCK(_mtxSSL);
      return -(SslNewFailed);
    } else {
      if (hostname) {
        if (!SSL_set_tlsext_host_name(_ssl, hostname)) {
          LOG_ERROR("Error setting SNI host name");
        } else {
          LOG_INFO("Set SNI %s success", hostname);
        }
      }
    }

    ret = SSL_set_fd(_ssl, socketFd);
    if (ret == 0) {
      memset(_errorMsg, 0x0, MaxSslErrorLength);
      const char *SSL_set_fd_ret = "return of SSL_set_fd: ";
      const int SSL_set_fd_str_size = strnlen(SSL_set_fd_ret, 24);
      memcpy(_errorMsg, SSL_set_fd_ret, SSL_set_fd_str_size);
      ERR_error_string_n(ERR_get_error(), _errorMsg + SSL_set_fd_str_size,
                         MaxSslErrorLength - SSL_set_fd_str_size - 1);
      LOG_ERROR("SSL(%p) Invoke SSL_set_fd failed:%s.", this, _errorMsg);
      MUTEX_UNLOCK(_mtxSSL);
      return -(SslSetFailed);
    }

    SSL_set_mode(_ssl, SSL_MODE_ENABLE_PARTIAL_WRITE |
                           SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER |
                           SSL_MODE_AUTO_RETRY);

    SSL_set_connect_state(_ssl);
  } else {
    // LOG_DEBUG("SSL has existed.");
  }

  ret = SSL_connect(_ssl);
  if (ret < 0) {
    int sslError = SSL_get_error(_ssl, ret);

    /* err == SSL_ERROR_ZERO_RETURN
       "SSL_connect: close notify received from peer" */
    // sslError == SSL_ERROR_WANT_X509_LOOKUP
    // SSL_ERROR_SYSCALL
    if (sslError == SSL_ERROR_WANT_READ || sslError == SSL_ERROR_WANT_WRITE) {
      // LOG_DEBUG("sslHandshake continue.");
      MUTEX_UNLOCK(_mtxSSL);
      return sslError;
    } else if (sslError == SSL_ERROR_SYSCALL) {
      int errno_code = utility::getLastErrorCode();
      LOG_INFO("SSL(%p) SSL connect error_syscall failed, errno:%d.", this,
               errno_code);

      if (NLS_ERR_CONNECT_RETRIABLE(errno_code) ||
          NLS_ERR_RW_RETRIABLE(errno_code)) {
        MUTEX_UNLOCK(_mtxSSL);
        return SSL_ERROR_WANT_READ;
      } else if (errno_code == 0) {
        LOG_DEBUG("SSL(%p) SSL connect syscall success.", this);
        MUTEX_UNLOCK(_mtxSSL);
        return Success;
      } else {
        MUTEX_UNLOCK(_mtxSSL);
        return -(SslConnectFailed);
      }
    } else {
      memset(_errorMsg, 0x0, MaxSslErrorLength);
      const char *SSL_connect_ret = "return of SSL_connect: ";
      const int SSL_connect_str_size = strnlen(SSL_connect_ret, 64);
      memcpy(_errorMsg, SSL_connect_ret, SSL_connect_str_size);
      ERR_error_string_n(ERR_get_error(), _errorMsg + SSL_connect_str_size,
                         MaxSslErrorLength - SSL_connect_str_size - 1);
      LOG_ERROR("SSL(%p) SSL connect failed:%s.", this, _errorMsg);
      MUTEX_UNLOCK(_mtxSSL);
      this->sslClose();
      return -(SslConnectFailed);
    }
  } else {
    // LOG_DEBUG("sslHandshake success.");
    MUTEX_UNLOCK(_mtxSSL);
    return Success;
  }

  MUTEX_UNLOCK(_mtxSSL);
  return Success;
}