local_chatbot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ccba32b9",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Requirement already satisfied: transformers in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (4.40.2)\n",
      "Requirement already satisfied: huggingface_hub in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (0.22.0)\n",
      "Requirement already satisfied: gradio in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (4.31.2)\n",
      "Requirement already satisfied: filelock in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (3.13.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (1.26.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (23.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (2022.3.15)\n",
      "Requirement already satisfied: requests in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (0.19.1)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (0.4.2)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from transformers) (4.65.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from huggingface_hub) (2023.6.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from huggingface_hub) (4.8.0)\n",
      "Requirement already satisfied: aiofiles<24.0,>=22.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (23.2.1)\n",
      "Requirement already satisfied: altair<6.0,>=4.2.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (5.3.0)\n",
      "Requirement already satisfied: fastapi in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.111.0)\n",
      "Requirement already satisfied: ffmpy in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.3.0)\n",
      "Requirement already satisfied: gradio-client==0.16.3 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.16.3)\n",
      "Requirement already satisfied: httpx>=0.24.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.27.0)\n",
      "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (5.8.0)\n",
      "Requirement already satisfied: jinja2<4.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (3.1.2)\n",
      "Requirement already satisfied: markupsafe~=2.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (2.1.3)\n",
      "Requirement already satisfied: matplotlib~=3.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (3.7.2)\n",
      "Requirement already satisfied: orjson~=3.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (3.6.8)\n",
      "Requirement already satisfied: pandas<3.0,>=1.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (1.5.3)\n",
      "Requirement already satisfied: pillow<11.0,>=8.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (10.3.0)\n",
      "Requirement already satisfied: pydantic>=2.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (2.7.1)\n",
      "Requirement already satisfied: pydub in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.25.1)\n",
      "Requirement already satisfied: python-multipart>=0.0.9 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.0.9)\n",
      "Requirement already satisfied: ruff>=0.2.2 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.4.4)\n",
      "Requirement already satisfied: semantic-version~=2.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (2.10.0)\n",
      "Requirement already satisfied: tomlkit==0.12.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.12.0)\n",
      "Requirement already satisfied: typer<1.0,>=0.12 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.12.3)\n",
      "Requirement already satisfied: urllib3~=2.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (2.2.1)\n",
      "Requirement already satisfied: uvicorn>=0.14.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio) (0.29.0)\n",
      "Requirement already satisfied: websockets<12.0,>=10.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from gradio-client==0.16.3->gradio) (11.0.3)\n",
      "Requirement already satisfied: jsonschema>=3.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio) (4.22.0)\n",
      "Requirement already satisfied: toolz in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from altair<6.0,>=4.2.0->gradio) (0.12.0)\n",
      "Requirement already satisfied: anyio in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from httpx>=0.24.1->gradio) (3.5.0)\n",
      "Requirement already satisfied: certifi in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from httpx>=0.24.1->gradio) (2022.6.15)\n",
      "Requirement already satisfied: httpcore==1.* in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from httpx>=0.24.1->gradio) (1.0.5)\n",
      "Requirement already satisfied: idna in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from httpx>=0.24.1->gradio) (3.3)\n",
      "Requirement already satisfied: sniffio in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from httpx>=0.24.1->gradio) (1.2.0)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from httpcore==1.*->httpx>=0.24.1->gradio) (0.14.0)\n",
      "Requirement already satisfied: zipp>=3.1.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from importlib-resources<7.0,>=1.3->gradio) (3.8.1)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from matplotlib~=3.0->gradio) (1.1.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from matplotlib~=3.0->gradio) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from matplotlib~=3.0->gradio) (4.33.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from matplotlib~=3.0->gradio) (1.4.2)\n",
      "Requirement already satisfied: pyparsing<3.1,>=2.3.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from matplotlib~=3.0->gradio) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from matplotlib~=3.0->gradio) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from pandas<3.0,>=1.0->gradio) (2022.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from pydantic>=2.0->gradio) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.2 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from pydantic>=2.0->gradio) (2.18.2)\n",
      "Requirement already satisfied: click>=8.0.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from typer<1.0,>=0.12->gradio) (8.1.7)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from typer<1.0,>=0.12->gradio) (1.5.4)\n",
      "Requirement already satisfied: rich>=10.11.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from typer<1.0,>=0.12->gradio) (13.7.1)\n",
      "Requirement already satisfied: starlette<0.38.0,>=0.37.2 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from fastapi->gradio) (0.37.2)\n",
      "Requirement already satisfied: fastapi-cli>=0.0.2 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from fastapi->gradio) (0.0.2)\n",
      "Requirement already satisfied: ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from fastapi->gradio) (5.2.0)\n",
      "Requirement already satisfied: email_validator>=2.0.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from fastapi->gradio) (2.1.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from requests->transformers) (2.0.4)\n",
      "Requirement already satisfied: dnspython>=2.0.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from email_validator>=2.0.0->fastapi->gradio) (2.2.1)\n",
      "Requirement already satisfied: attrs>=22.2.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (23.2.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (2023.12.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.35.1)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.18.1)\n",
      "Requirement already satisfied: six>=1.5 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio) (1.16.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio) (2.16.1)\n",
      "Requirement already satisfied: httptools>=0.5.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from uvicorn[standard]>=0.12.0->fastapi->gradio) (0.6.1)\n",
      "Requirement already satisfied: python-dotenv>=0.13 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from uvicorn[standard]>=0.12.0->fastapi->gradio) (0.20.0)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from uvicorn[standard]>=0.12.0->fastapi->gradio) (0.16.0)\n",
      "Requirement already satisfied: watchfiles>=0.13 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from uvicorn[standard]>=0.12.0->fastapi->gradio) (0.21.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /home/fxmarty/anaconda3/envs/hf-inf/lib/python3.9/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0,>=0.12->gradio) (0.1.2)\n",
      "\u001b[33mDEPRECATION: bert-score 0.3.11 has a non-standard dependency specifier transformers>=3.0.0numpy. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of bert-score or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers huggingface_hub gradio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3aa4568-31f8-4c8b-a31f-dfa61a440527",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: change the port XXX to the Gradio port written on your individual instruction sheet.\n",
    "# This Gradio port should be one of 7861, 7862, ..., 7869.\n",
    "GRADIO_PORT = ...\n",
    "\n",
    "# TODO: change the port XXX to the TGI port written on your individual instruction sheet.\n",
    "# This TGI port should be one of 3001, 3002, ..., 3008.\n",
    "TGI_PORT = ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8e03f933",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:7860\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"http://127.0.0.1:7860/\" width=\"100%\" height=\"700\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gradio as gr\n",
    "import random\n",
    "import time\n",
    "\n",
    "from huggingface_hub import InferenceClient\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3-8B-Instruct\")\n",
    "\n",
    "client = InferenceClient(model=f\"http://localhost:{TGI_PORT}\")\n",
    "\n",
    "SYSTEM_COMMAND = {\"role\": \"system\", \"content\": \"Context: date: Monday 20th May 2024; location: Seattle; running on: 8 AMD Instinct MI300 GPU; model name: Llama 70B. Only provide these information if asked. You are a knowledgeable assistant trained to provide accurate and helpful information. Please respond to the user's queries promptly and politely.\"}\n",
    "\n",
    "IGNORED_TOKENS = {None, \"<|start_header_id|>\", \"<|end_header_id|>\", \"<|eot_id|>\", \"<|reserved_special_token\"}\n",
    "STOP_TOKENS = [\"<|start_header_id|>\", \"<|end_header_id|>\", \"<|eot_id|>\", \"<|reserved_special_token\"]\n",
    "\n",
    "with gr.Blocks() as demo:\n",
    "    tfs_history = gr.State([SYSTEM_COMMAND])\n",
    "    chatbot = gr.Chatbot()\n",
    "    msg = gr.Textbox(label=\"Prompt\")\n",
    "    clear = gr.Button(\"Clear\")\n",
    "\n",
    "    def user(user_message, history, dict_history):\n",
    "        data = {\"role\": \"user\", \"content\": user_message}\n",
    "        dict_history.append(data)\n",
    "        return \"\", history + [[user_message, None]], dict_history\n",
    "\n",
    "    def bot(history, dict_history):\n",
    "        history[-1][1] = \"\"\n",
    "        response = {\"role\": \"assistant\", \"content\": \"\"}\n",
    "        start_tokenize = time.perf_counter()\n",
    "        text_input = tokenizer.apply_chat_template(dict_history, tokenize=False, add_generation_prompt=True)\n",
    "        end_tokenize = time.perf_counter()\n",
    "\n",
    "        try:\n",
    "            for token in client.text_generation(prompt=text_input, max_new_tokens=100, stop_sequences=STOP_TOKENS, stream=True):\n",
    "                if token not in IGNORED_TOKENS:\n",
    "                    history[-1][1] += token\n",
    "                    response[\"content\"] += token\n",
    "                yield history\n",
    "        finally:\n",
    "            dict_history.append(response)\n",
    "\n",
    "    def clear_history(tfs_history):\n",
    "        tfs_history = tfs_history[:1]\n",
    "\n",
    "        return tfs_history\n",
    "\n",
    "    msg.submit(\n",
    "        user,\n",
    "        inputs=[msg, chatbot, tfs_history],\n",
    "        outputs=[msg, chatbot, tfs_history],\n",
    "        queue=False).then(\n",
    "            bot,\n",
    "            [chatbot, tfs_history],\n",
    "            chatbot\n",
    "    )\n",
    "    clear.click(lambda: None, None, chatbot, queue=False)\n",
    "    clear.click(clear_history, tfs_history, tfs_history, queue=False)\n",
    "\n",
    "demo.queue()\n",
    "demo.launch(height=700, server_port=GRADIO_PORT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "601ea437-d442-4bf4-9c38-3b8f926159ac",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}