int webclient_perform()

in netutils/webclient/webclient.c [1176:2208]


int webclient_perform(FAR struct webclient_context *ctx)
{
  struct wget_s *ws;
  struct timeval tv;
  char *dest;
  char *ep;
  struct webclient_conn_s *conn;
  FAR const struct webclient_tls_ops *tls_ops = ctx->tls_ops;
  FAR const char *method = ctx->method;
  FAR void *tls_ctx = ctx->tls_ctx;
  unsigned int i;
  int len;
  int ret;

#ifdef CONFIG_DEBUG_ASSERTIONS
  DEBUGASSERT(ctx->state == WEBCLIENT_CONTEXT_STATE_INITIALIZED ||
              (ctx->state == WEBCLIENT_CONTEXT_STATE_IN_PROGRESS &&
               (ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0));
#endif

#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
  if (ctx->unix_socket_path != NULL && ctx->proxy != NULL)
    {
      nerr("ERROR: proxy with AF_LOCAL is not implemented");
      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
      return -ENOTSUP;
    }
#endif

  /* Initialize the state structure */

  if (ctx->ws == NULL)
    {
      ws = calloc(1, sizeof(struct wget_s));
      if (!ws)
        {
          _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
          return -errno;
        }

      ws->conn = calloc(1, sizeof(struct webclient_conn_s));
      if (!ws->conn)
        {
          free_ws(ws);
          _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
          return -errno;
        }

      ws->buffer = ctx->buffer;
      ws->buflen = ctx->buflen;

      /* Parse the hostname (with optional port number) and filename
       * from the URL.
       */

      if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) == 0)
        {
          ret = parseurl(ctx->url, &ws->target, false);
          if (ret != 0)
            {
              nwarn("WARNING: Malformed URL: %s\n", ctx->url);
              free_ws(ws);
              _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
              return ret;
            }
        }

      if (ctx->proxy != NULL)
        {
          /* Note: reject a proxy string w/o port number specified.
           * It's better to be explicit because the default number varies
           * among HTTP client implementations.
           * (80, 1080, 3128, 8080, ...)
           */

          ret = parseurl(ctx->proxy, &ws->proxy, true);
          if (ret != 0)
            {
              nerr("ERROR: Malformed proxy setting: %s\n", ctx->proxy);
              free_ws(ws);
              _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
              return ret;
            }

          if (strcmp(ws->proxy.scheme, "http") ||
              strcmp(ws->proxy.filename, "/"))
            {
              nerr("ERROR: Unsupported proxy setting: %s\n", ctx->proxy);
              free_ws(ws);
              _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
              return -ENOTSUP;
            }
        }

      ws->state = WEBCLIENT_STATE_SOCKET;
      ctx->ws = ws;
    }

  ws = ctx->ws;

  ninfo("hostname='%s' filename='%s'\n", ws->target.hostname,
        ws->target.filename);

  /* The following sequence may repeat indefinitely if we are redirected */

  conn = ws->conn;
  do
    {
      if (ws->state == WEBCLIENT_STATE_SOCKET)
        {
          if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0)
            {
              conn->tls = false;
            }
          else if (!strcmp(ws->target.scheme, "https") && tls_ops != NULL)
            {
              conn->tls = true;
              conn->tls_ops = tls_ops;
              conn->tls_ctx = ctx->tls_ctx;
            }
          else if (!strcmp(ws->target.scheme, "http"))
            {
              conn->tls = false;
            }
          else
            {
              nerr("ERROR: unsupported scheme: %s\n", ws->target.scheme);
              free_ws(ws);
              _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
              return -ENOTSUP;
            }

          /* Re-initialize portions of the state structure that could have
           * been left from the previous time through the loop and should not
           * persist with the new connection.
           */

          ws->httpstatus = HTTPSTATUS_NONE;
          ws->offset     = 0;
          ws->datend     = 0;
          ws->ndx        = 0;
          ws->redirected = 0;

          if (conn->tls)
            {
#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              if (ctx->unix_socket_path != NULL)
                {
                  nerr("ERROR: TLS on AF_LOCAL socket is not implemented\n");
                  free_ws(ws);
                  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
                  return -ENOTSUP;
                }
#endif

              if (ctx->proxy != NULL)
                {
                  FAR struct webclient_context *tunnel;

                  DEBUGASSERT(ws->tunnel == NULL);

                  if (tls_ops->init_connection == NULL)
                    {
                      nerr("ERROR: TLS over proxy is not implemented\n");
                      ret = -ENOTSUP;
                      goto errout_with_errno;
                    }

                  /* Create a temporary context to establish a tunnel. */

                  ws->tunnel = tunnel = calloc(1, sizeof(*ws->tunnel));
                  if (tunnel == NULL)
                    {
                      ret = -ENOMEM;
                      goto errout_with_errno;
                    }

                  webclient_set_defaults(tunnel);
                  tunnel->method = "CONNECT";
                  tunnel->flags |= WEBCLIENT_FLAG_TUNNEL;
                  tunnel->tunnel_target_host = ws->target.hostname;
                  tunnel->tunnel_target_port = ws->target.port;
                  tunnel->proxy = ctx->proxy;
                  tunnel->proxy_headers = ctx->proxy_headers;
                  tunnel->proxy_nheaders = ctx->proxy_nheaders;

                  /* Inherit some configurations.
                   *
                   * Revisit: should there be independent configurations?
                   */

                  tunnel->protocol_version = ctx->protocol_version;
                  if ((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0)
                    {
                      tunnel->flags |= WEBCLIENT_FLAG_NON_BLOCKING;
                    }

                  /* Abuse the buffer of the original request.
                   * It's safe with the current usage.
                   */

                  tunnel->buffer = ctx->buffer;
                  tunnel->buflen = ctx->buflen;
                }
            }
          else
            {
              int domain;

#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              if (ctx->unix_socket_path != NULL)
                {
                  domain = PF_LOCAL;
                }
              else
#endif
                {
                  domain = PF_INET;
                }

              /* Create a socket */

              conn->sockfd = socket(domain, SOCK_STREAM, 0);
              if (conn->sockfd < 0)
                {
                  ret = -errno;
                  nerr("ERROR: socket failed: %d\n", errno);
                  goto errout_with_errno;
                }

              ws->need_conn_close = true;

              if ((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0)
                {
                  int flags = fcntl(conn->sockfd, F_GETFL, 0);
                  ret = fcntl(conn->sockfd, F_SETFL, flags | O_NONBLOCK);
                  if (ret == -1)
                    {
                      ret = -errno;
                      nerr("ERROR: F_SETFL failed: %d\n", ret);
                      goto errout_with_errno;
                    }
                }
              else
                {
                  /* Set send and receive timeout values */

                  tv.tv_sec  = ctx->timeout_sec;
                  tv.tv_usec = 0;

                  /* Check return value one by one */

                  ret = setsockopt(conn->sockfd, SOL_SOCKET, SO_RCVTIMEO,
                                   &tv, sizeof(struct timeval));
                  if (ret != 0)
                    {
                      ret = -errno;
                      nerr("ERROR: setsockopt failed: %d\n", ret);
                      goto errout_with_errno;
                    }

                  ret = setsockopt(conn->sockfd, SOL_SOCKET, SO_SNDTIMEO,
                                   &tv, sizeof(struct timeval));
                  if (ret != 0)
                    {
                      ret = -errno;
                      nerr("ERROR: setsockopt failed: %d\n", ret);
                      goto errout_with_errno;
                    }
                }
            }

          ws->state = WEBCLIENT_STATE_CONNECT;
        }

      if (ws->state == WEBCLIENT_STATE_CONNECT)
        {
          if (ws->tunnel != NULL)
            {
              ret = webclient_perform(ws->tunnel);
              if (ret == 0)
                {
                  unsigned int http_status = ws->tunnel->http_status;

                  if (http_status / 100 != 2)
                    {
                      nerr("HTTP-level error %u on tunnel attempt\n",
                           http_status);

                      /* 407 Proxy Authentication Required */

                      if (http_status == 407)
                        {
                          ret = -EPERM;
                        }
                      else
                        {
                          ret = -EIO;
                        }
                    }
                }

              if (ret == 0)
                {
                  FAR struct webclient_conn_s *tunnel_conn;

                  webclient_get_tunnel(ws->tunnel, &tunnel_conn);
                  DEBUGASSERT(tunnel_conn != NULL);
                  DEBUGASSERT(!tunnel_conn->tls);
                  free(ws->tunnel);
                  ws->tunnel = NULL;

                  if (conn->tls)
                    {
                      /* Revisit: tunnel_conn here should have
                       * timeout configured already.
                       * Configuring it again here is redundant.
                       */

                      ret = tls_ops->init_connection(tls_ctx,
                                                     tunnel_conn,
                                                     ws->target.hostname,
                                                     ctx->timeout_sec,
                                                     &conn->tls_conn);
                      if (ret == 0)
                        {
                          /* Note: tunnel_conn has been consumed by
                           * tls_ops->init_connection
                           */

                          ws->need_conn_close = true;
                        }
                      else
                        {
                          /* Note: restarting tls_ops->init_connection
                           * is not implemented
                           */

                          DEBUGASSERT(ret != -EAGAIN &&
                                      ret != -EINPROGRESS &&
                                      ret != -EALREADY);
                          webclient_conn_close(tunnel_conn);
                          webclient_conn_free(tunnel_conn);
                        }
                    }
                  else
                    {
                      conn->sockfd = tunnel_conn->sockfd;
                      ws->need_conn_close = true;
                      webclient_conn_free(tunnel_conn);
                    }
                }
            }
          else if (conn->tls)
            {
              char port_str[sizeof("65535")];

#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              if (ctx->unix_socket_path != NULL)
                {
                  nerr("ERROR: TLS on AF_LOCAL socket is not implemented\n");
                  free_ws(ws);
                  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
                  return -ENOTSUP;
                }
#endif

              snprintf(port_str, sizeof(port_str), "%u", ws->target.port);
              ret = tls_ops->connect(tls_ctx, ws->target.hostname, port_str,
                                     ctx->timeout_sec,
                                     &conn->tls_conn);
              if (ret == 0)
                {
                  ws->need_conn_close = true;
                }
            }
          else
            {
#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              struct sockaddr_un server_un;
#endif
              struct sockaddr_in server_in;
              const struct sockaddr *server_address;
              socklen_t server_address_len;

#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              if (ctx->unix_socket_path != NULL)
                {
                  memset(&server_un, 0, sizeof(server_un));
                  server_un.sun_family = AF_LOCAL;
                  strlcpy(server_un.sun_path, ctx->unix_socket_path,
                          sizeof(server_un.sun_path));
#if !defined(__NuttX__) && !defined(__linux__)
                  server_un.sun_len = SUN_LEN(&server_un);
#endif
                  server_address = (const struct sockaddr *)&server_un;
                  server_address_len = sizeof(server_un);
                }
              else
#endif
                {
                  /* Get the server address from the host name */

                  FAR struct wget_target_s *target;

                  if (ctx->proxy != NULL)
                    {
                      target = &ws->proxy;
                    }
                  else
                    {
                      target = &ws->target;
                    }

                  server_in.sin_family = AF_INET;
                  server_in.sin_port   = htons(target->port);
                  ret = wget_gethostip(target->hostname,
                                       &server_in.sin_addr);
                  if (ret < 0)
                    {
                      /* Could not resolve host (or malformed IP address) */

                      nwarn("WARNING: Failed to resolve hostname\n");
                      free_ws(ws);
                      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
                      return -EHOSTUNREACH;
                    }

                  server_address = (const struct sockaddr *)&server_in;
                  server_address_len = sizeof(struct sockaddr_in);
                }

              /* Connect to server.  First we have to set some fields in the
               * 'server' address structure.  The system will assign me an
               * arbitrary local port that is not in use.
               */

              while (true)
                {
                  ret = connect(conn->sockfd, server_address,
                                server_address_len);
                  if (ret == -1)
                    {
                      ret = -errno;
                      DEBUGASSERT(ret < 0);
                      if (ret == -EINTR)
                        {
                          continue;
                        }

                      if (ret == -EISCONN)
                        {
                          ret = 0;
                        }
                    }
                  break;
                }
            }

          if (ret < 0)
            {
              nerr("ERROR: connect failed: %d\n", errno);
              goto errout_with_errno;
            }

          ws->state = WEBCLIENT_STATE_PREPARE_REQUEST;
        }

      if (ws->state == WEBCLIENT_STATE_PREPARE_REQUEST)
        {
          /* Send the request */

          dest = ws->buffer;
          ep = ws->buffer + ws->buflen;
          dest = append(dest, ep, method);
          dest = append(dest, ep, " ");

          if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0)
            {
              /* Use authority-form for a tunnel
               *
               * https://datatracker.ietf.org/doc/html/rfc7231#section-4.3.6
               * https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3
               */

              char port_str[sizeof("65535")];

              dest = append(dest, ep, ctx->tunnel_target_host);
              dest = append(dest, ep, ":");
              snprintf(port_str, sizeof(port_str), "%u",
                       ctx->tunnel_target_port);
              dest = append(dest, ep, port_str);
            }
          else if (ctx->proxy != NULL)
            {
              /* Use absolute-form for a proxy
               *
               * https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
               */

              char port_str[sizeof("65535")];

              dest = append(dest, ep, ws->target.scheme);
              dest = append(dest, ep, "://");
              dest = append(dest, ep, ws->target.hostname);
              dest = append(dest, ep, ":");
              snprintf(port_str, sizeof(port_str), "%u", ws->target.port);
              dest = append(dest, ep, port_str);
              dest = append(dest, ep, ws->target.filename);
            }
          else
            {
              /* Otherwise, use origin-form */

#ifndef WGET_USE_URLENCODE
              dest = append(dest, ep, ws->target.filename);
#else
              /* TODO: should we use wget_urlencode_strcpy? */

              dest = append(dest, ep, ws->target.filename);
#endif
            }

          dest = append(dest, ep, " ");
          if (ctx->protocol_version == WEBCLIENT_PROTOCOL_VERSION_HTTP_1_0)
            {
              dest = append(dest, ep, g_http10);
            }
          else if (ctx->protocol_version ==
                   WEBCLIENT_PROTOCOL_VERSION_HTTP_1_1)
            {
              dest = append(dest, ep, g_http11);
            }
          else
            {
              ret = -EINVAL;
              goto errout_with_errno;
            }

          dest = append(dest, ep, g_httpcrnl);

          /* Note about proxy and Host header:
           *
           * https://datatracker.ietf.org/doc/html/rfc7230#section-5.4
           * > A client MUST send a Host header field in an HTTP/1.1
           * > request even if the request-target is in the absolute-form,
           * > since this allows the Host information to be forwarded
           * > through ancient HTTP/1.0 proxies that might not have
           * > implemented Host.
           *
           * > When a proxy receives a request with an absolute-form of
           * > request-target, the proxy MUST ignore the received Host
           * > header field (if any) and instead replace it with the host
           * > information of the request-target.
           */

          dest = append(dest, ep, g_httphost);
          dest = append(dest, ep, ws->target.hostname);
          dest = append(dest, ep, g_httpcrnl);

          /* Append user-specified headers.
           *
           * - For non-proxied request,
           *   only send "headers".
           *
           * - For proxied request, (traditional http proxy)
           *   send both of "headers" and "proxy_headers".
           *
           * - For tunneling request, (WEBCLIENT_FLAG_TUNNEL)
           *   only send "proxy_headers".
           */

          if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) == 0)
            {
              for (i = 0; i < ctx->nheaders; i++)
                {
                  dest = append(dest, ep, ctx->headers[i]);
                  dest = append(dest, ep, g_httpcrnl);
                }
            }

          if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0 ||
              ctx->proxy != NULL)
            {
              for (i = 0; i < ctx->proxy_nheaders; i++)
                {
                  dest = append(dest, ep, ctx->proxy_headers[i]);
                  dest = append(dest, ep, g_httpcrnl);
                }
            }

          if (ctx->bodylen)
            {
              char post_size[sizeof("18446744073709551615")];

              dest = append(dest, ep, g_httpcontsize);
              snprintf(post_size, sizeof(post_size), "%zu", ctx->bodylen);
              dest = append(dest, ep, post_size);
              dest = append(dest, ep, g_httpcrnl);
            }

          if (ctx->protocol_version == WEBCLIENT_PROTOCOL_VERSION_HTTP_1_1)
            {
              /* We don't implement persistect connections. */

              dest = append(dest, ep, g_httpconn_close);
              dest = append(dest, ep, g_httpcrnl);
            }

          dest = append(dest, ep, g_httpuseragentfields);

          if (dest == NULL)
            {
              ret = -E2BIG;
              goto errout_with_errno;
            }

          len = dest - ws->buffer;

          ws->state = WEBCLIENT_STATE_SEND_REQUEST;
          ws->state_offset = 0;
          ws->state_len = len;
        }

      if (ws->state == WEBCLIENT_STATE_SEND_REQUEST)
        {
          ssize_t ssz;

          ssz = webclient_conn_send(conn,
                                    ws->buffer + ws->state_offset,
                                    ws->state_len);
          if (ssz < 0)
            {
              ret = ssz;
              nerr("ERROR: send failed: %d\n", -ret);
              goto errout_with_errno;
            }

          if (ssz >= 0)
            {
              ws->state_offset += ssz;
              ws->state_len -= ssz;
              if (ws->state_len == 0)
                {
                  ws->state = WEBCLIENT_STATE_SEND_REQUEST_BODY;
                  ws->state_offset = 0;
                  ws->state_len = ctx->bodylen;
                  ws->data_buffer = NULL;
                  ninfo("Sending %zu bytes request body\n", ws->state_len);
                }
            }
        }

      if (ws->state == WEBCLIENT_STATE_SEND_REQUEST_BODY)
        {
          /* In this state,
           *
           * ws->data_buffer  the data given by the user
           * ws->data_offset  the byte offset in the entire body
           * ws->data_len     the byte size of the data in ws->data_buffer
           * ws->state_offset the send offset in ws->data_buffer
           * ws->state_len    the number of remaining bytes to send (total)
           */

          if (ws->state_len == 0)
            {
              ninfo("Finished sending request body\n");
              ws->state = WEBCLIENT_STATE_STATUSLINE;
            }
          else if (ws->data_buffer == NULL)
            {
              FAR const void *input_buffer;
              size_t input_buffer_size = ws->buflen;

              size_t todo = ws->state_len;
              if (input_buffer_size > todo)
                {
                  input_buffer_size = todo;
                }

              input_buffer = ws->buffer;
              ret = ctx->body_callback(ws->buffer,
                                       &input_buffer_size,
                                       &input_buffer,
                                       todo,
                                       ctx->body_callback_arg);
              if (ret < 0)
                {
                  nerr("ERROR: body_callback failed: %d\n", -ret);
                  goto errout_with_errno;
                }

              ninfo("Got %zu bytes body chunk / total remaining %zu bytes\n",
                    input_buffer_size, ws->state_len);
              ws->data_buffer = input_buffer;
              ws->data_len = input_buffer_size;
              ws->state_offset = 0;
            }
          else
            {
              size_t bytes_to_send = ws->data_len - ws->state_offset;

              DEBUGASSERT(bytes_to_send <= ws->state_len);
              ssize_t ssz = webclient_conn_send(conn,
                                                (char *)ws->data_buffer +
                                                ws->state_offset,
                                                bytes_to_send);
              if (ssz < 0)
                {
                  ret = ssz;
                  nerr("ERROR: send failed: %d\n", -ret);
                  goto errout_with_errno;
                }

              ninfo("Sent %zd bytes request body at offset %ju "
                    "in the %zu bytes chunk, "
                    "total remaining %zu bytes\n",
                    ssz,
                    (uintmax_t)ws->state_offset,
                    ws->data_len,
                    ws->state_len);
              ws->state_len -= ssz;
              ws->state_offset += ssz;
              DEBUGASSERT((size_t)ws->state_offset <= ws->data_len);
              if ((size_t)ws->state_offset == ws->data_len)
                {
                  ws->data_buffer = NULL;
                }
            }
        }

      /* Now loop to get the file sent in response to the GET.  This
       * loop continues until either we read the end of file (nbytes == 0)
       * or until we detect that we have been redirected.
       */

      if (ws->state == WEBCLIENT_STATE_STATUSLINE ||
          ws->state == WEBCLIENT_STATE_HEADERS ||
          ws->state == WEBCLIENT_STATE_DATA ||
          ws->state == WEBCLIENT_STATE_CHUNKED_HEADER ||
          ws->state == WEBCLIENT_STATE_CHUNKED_DATA)
        {
          for (; ; )
            {
              if (ws->datend - ws->offset == 0)
                {
                  size_t want = ws->buflen;
                  ssize_t ssz;

                  ninfo("Reading new data\n");
                  if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0)
                    {
                      /* When tunnelling, we want to avoid troubles
                       * with reading the starting payload of the tunnelled
                       * protocol here, in case it's a server-speaks-first
                       * protocol.
                       */

                      want = 1;
                    }

                  ssz = webclient_conn_recv(conn, ws->buffer, want);
                  if (ssz < 0)
                    {
                      ret = ssz;
                      nerr("ERROR: recv failed: %d\n", -ret);
                      goto errout_with_errno;
                    }
                  else if (ssz == 0)
                    {
                      if (ws->state != WEBCLIENT_STATE_DATA &&
                          ws->state != WEBCLIENT_STATE_WAIT_CLOSE)
                        {
                          nerr("Connection lost unexpectedly\n");
                          ret = -ECONNABORTED;
                          goto errout_with_errno;
                        }

                      if ((ws->internal_flags &
                           WGET_FLAG_GOT_CONTENT_LENGTH) != 0 &&
                          ws->expected_resp_body_len !=
                          ws->received_body_len)
                        {
                          nerr("Unexpected response body length: "
                               "%ju != %ju\n",
                               ws->expected_resp_body_len,
                               ws->received_body_len);
                          ret = -EPROTO;
                          goto errout_with_errno;
                        }

                      ninfo("Connection lost\n");
                      ws->state = WEBCLIENT_STATE_CLOSE;
                      ws->redirected = 0;
                      break;
                    }

                  ninfo("Got %zd bytes data\n", ssz);
                  ws->offset = 0;
                  ws->datend = ssz;
                }

              /* Handle initial parsing of the status line */

              if (ws->state == WEBCLIENT_STATE_STATUSLINE)
                {
                  ret = wget_parsestatus(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              /* Parse the HTTP data */

              if (ws->state == WEBCLIENT_STATE_HEADERS)
                {
                  ret = wget_parseheaders(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              /* Parse the chunk header */

              if (ws->state == WEBCLIENT_STATE_CHUNKED_HEADER)
                {
                  ret = wget_parsechunkheader(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              if (ws->state == WEBCLIENT_STATE_CHUNKED_ENDDATA)
                {
                  ret = wget_parsechunkenddata(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              if (ws->state == WEBCLIENT_STATE_CHUNKED_TRAILER)
                {
                  ret = wget_parsechunktrailer(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              if (ws->state == WEBCLIENT_STATE_WAIT_CLOSE)
                {
                  uintmax_t received = ws->datend - ws->offset;
                  if (received != 0)
                    {
                      nerr("Unexpected %ju bytes data received", received);
                      ret = -EPROTO;
                      goto errout_with_errno;
                    }
                }

              /* Dispose of the data payload */

              if (ws->state == WEBCLIENT_STATE_DATA ||
                  ws->state == WEBCLIENT_STATE_CHUNKED_DATA)
                {
                  if (ws->httpstatus != HTTPSTATUS_MOVED)
                    {
                      uintmax_t received = ws->datend - ws->offset;
                      FAR char *orig_buffer = ws->buffer;
                      int orig_buflen = ws->buflen;

                      if ((ws->internal_flags & WGET_FLAG_GOT_LOCATION) != 0)
                        {
                          nwarn("WARNING: Unexpected Location header\n");
                        }

                      if (ws->state == WEBCLIENT_STATE_CHUNKED_DATA)
                        {
                          uintmax_t chunk_left =
                              ws->chunk_len - ws->chunk_received;

                          if (received > chunk_left)
                            {
                              received = chunk_left;
                            }

                          ws->chunk_received += received;
                        }

                      ninfo("Processing resp body %ju - %ju\n",
                            ws->received_body_len,
                            ws->received_body_len + received);
                      ws->received_body_len += received;

                      /* Let the client decide what to do with the
                       * received file.
                       */

                      if (received == 0)
                        {
                          /* We don't have data to give to the client yet. */
                        }
                      else if (ctx->sink_callback)
                        {
                          ret = ctx->sink_callback(&ws->buffer, ws->offset,
                                                   ws->offset + received,
                                                   &ws->buflen,
                                                   ctx->sink_callback_arg);
                          if (ret != 0)
                            {
                              goto errout_with_errno;
                            }
                        }
                      else if (ctx->callback)
                        {
                          ctx->callback(&ws->buffer, ws->offset,
                                        ws->offset + received,
                                        &ws->buflen, ctx->sink_callback_arg);
                        }

                      ws->offset += received;

                      /* The buffer swapping API doesn't work for
                       * HTTP 1.1 chunked transfer because the buffer here
                       * might already contain the next chunk header.
                       */

                      if (ctx->protocol_version ==
                          WEBCLIENT_PROTOCOL_VERSION_HTTP_1_1)
                        {
                          if (orig_buffer != ws->buffer ||
                              orig_buflen != ws->buflen)
                            {
                              ret = -EINVAL;
                              goto errout_with_errno;
                            }
                        }

                      if (ws->state == WEBCLIENT_STATE_CHUNKED_DATA)
                        {
                          if (ws->chunk_len == ws->chunk_received)
                            {
                              ws->state = WEBCLIENT_STATE_CHUNKED_ENDDATA;
                              ws->ndx = 0;
                            }
                        }
                    }
                  else
                    {
                      if ((ws->internal_flags & WGET_FLAG_GOT_LOCATION) == 0)
                        {
                          nerr("ERROR: Redirect w/o Location header\n");
                          ret = -EPROTO;
                          goto errout_with_errno;
                        }

                      ws->nredirect++;
                      if (ws->nredirect > CONFIG_WEBCLIENT_MAX_REDIRECT)
                        {
                          nerr("ERROR: too many redirects (%u)\n",
                               ws->nredirect);
                          ret = -ELOOP;
                          goto errout_with_errno;
                        }

                      ws->state = WEBCLIENT_STATE_CLOSE;
                      ws->redirected = 1;
                      break;
                    }
                }

              if (ws->state == WEBCLIENT_STATE_TUNNEL_ESTABLISHED)
                {
                  break;
                }
            }
        }

      if (ws->state == WEBCLIENT_STATE_CLOSE)
        {
          webclient_conn_close(conn);
          ws->need_conn_close = false;
          if (ws->redirected)
            {
              ws->state = WEBCLIENT_STATE_SOCKET;
            }
          else
            {
              ws->state = WEBCLIENT_STATE_DONE;
            }
        }
    }
  while (ws->state != WEBCLIENT_STATE_DONE &&
         ws->state != WEBCLIENT_STATE_TUNNEL_ESTABLISHED);

  if (ws->state == WEBCLIENT_STATE_DONE)
    {
      free_ws(ws);
      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
    }
  else
    {
      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_TUNNEL_ESTABLISHED);
    }

  return OK;

errout_with_errno:
  if ((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0 &&
      (ret == -EAGAIN || ret == -EINPROGRESS || ret == -EALREADY))
    {
      if (ret == -EINPROGRESS || ret == -EALREADY)
        {
          conn->flags |= CONN_WANT_WRITE;
        }

      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_IN_PROGRESS);
      return -EAGAIN;
    }

  if (ws->need_conn_close)
    {
      webclient_conn_close(conn);
    }

  free_ws(ws);
  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
  return ret;
}