1_synthetic-qa-generation/auto-evolve-instruct/trial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import time\n",
    "import uuid\n",
    "import random\n",
    "import openai\n",
    "import markdown\n",
    "import textwrap\n",
    "from tqdm import tqdm\n",
    "from bs4 import BeautifulSoup\n",
    "from datasets import load_dataset\n",
    "from dotenv import load_dotenv\n",
    "from openai import AzureOpenAI, RateLimitError\n",
    "\n",
    "load_dotenv()  # take environment variables from .env.\n",
    "\n",
    "client = AzureOpenAI(\n",
    "    azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "    api_key        = os.getenv(\"AZURE_OPENAI_API_KEY\"),\n",
    "    api_version    = os.getenv(\"AZURE_OPENAI_API_VERSION\")\n",
    ")\n",
    "\n",
    "LANGUAGE = \"English\"\n",
    "MODEL_NAME = \"gpt-4o\"\n",
    "MODEL_NAME_FOR_ANSWER = \"gpt-4o\"\n",
    "\n",
    "MAX_NUMBER_OF_FIELDS = 1\n",
    "MAX_NUMBER_OF_SUBJECTS = 2\n",
    "MAX_NUMBER_OF_SUBTOPICS = 3\n",
    "MAX_NUMBER_OF_SESSION_NAME = 3\n",
    "NUM_ITERATIONS = 1\n",
    "NUM_QUESTIONS_PER_ITERATION = 3\n",
    "QUESTION_MAX_TOKENS = 256\n",
    "QUESTION_BACTH_SIZE = 3\n",
    "ANSWER_BACTH_SIZE = 3\n",
    "OUTPUT_DIR = \"./outputs\"\n",
    "UUID = str(uuid.uuid4())[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Initial Evolving Method\n",
    "import re\n",
    "import random\n",
    "def call_llm(prompt, model_name=\"gpt-4o\", max_tokens=150, **kwargs):\n",
    "\n",
    "    response = client.chat.completions.create(\n",
    "        model=model_name,\n",
    "        messages=[{\"role\": \"user\", \"content\": prompt}],\n",
    "        #response_format = {'type': \"json_object\"},\n",
    "        **kwargs    \n",
    "    )\n",
    "    #     \n",
    "    output = response.choices[0].message.content.strip()\n",
    "\n",
    "    return output\n",
    "\n",
    "def get_step_0_evolved_instruction(instruction, **kwargs):\n",
    "    prompt = f\"\"\"You are an Instruction Rewriter that rewrites the given #Instruction# into a more complex version. Please follow the steps below to rewrite the given \"#Instruction#\" into a more complex version.\n",
    "\n",
    "    ### Step 1: Please read the \"#Instruction#\" carefully and list all the possible methods to make this instruction more complex (to make it a bit harder for well-known AI assistants such as ChatGPT and GPT4 to handle). Please do not provide methods to change the language of the instruction!\n",
    "    ### Step 2: Please create a comprehensive plan based on the #Methods List# generated in Step 1 to make the #Instruction# more complex. The plan should include several methods from the #Methods List#.\n",
    "    ### Step 3: Please execute the plan step by step and provide the #Rewritten Instruction#. #Rewritten Instruction# can only add 10 to 20 words into the \"#Instruction#\".\n",
    "    ### Step 4: Please carefully review the #Rewritten Instruction# and identify any unreasonable parts. Ensure that the #Rewritten Instruction# is only a more complex version of the #Instruction#. Just provide the #Finally Rewritten Instruction# without any explanation.\n",
    "\n",
    "    Please reply strictly in the following format:\n",
    "\n",
    "    ### Step 1 #Methods List#:\n",
    "    ### Step 2 #Plan#:\n",
    "    ### Step 3 #Rewritten Instruction#:\n",
    "    ### Step 4 #Finally Rewritten Instruction#:\n",
    "\n",
    "    #Instruction#: {instruction}\n",
    "    \"\"\"\n",
    "    \n",
    "    output = call_llm(prompt, **kwargs)\n",
    "    rewritten_instruction = re.search(r\"Step 4 #Finally Rewritten Instruction#:(.*)\", output, re.DOTALL).group(1).strip()\n",
    "    return rewritten_instruction\n",
    "\n",
    "def get_step_1_evolved_instruction(instruction, **kwargs):\n",
    "    prompt = f\"\"\"You are an Instruction Rewriter that rewrites the given #Instruction# into a more complex version. Please follow the steps below to rewrite the given \"#Instruction#\" into a more complex version.\n",
    "\n",
    "    ### Step 1: Carefully read the initial instruction and identify all the elements involved - this includes variables, constants, operations, and conditions.\n",
    "    ### Step 2: Consider how each element could be made more complex. For variables, this could involve introducing more variables or making the existing variables dependent on others. For constants, consider changing them to variables or making them dependent on other factors. For operations, consider introducing more complex operations or multiple steps. For conditions, consider adding more conditions or making the existing conditions more complex.\n",
    "    ### Step 3: Formulate a plan to integrate these complexities into the instruction. Ensure that the changes are coherent and relevant to the initial problem context. The plan should not just randomly add complexity but should make the problem more interesting or challenging in a meaningful way.\n",
    "    ### Step 4: Rewrite the instruction according to the plan. Ensure that the rewritten instruction is still understandable and that it accurately represents the initial problem context. The rewritten instruction should only add 10 to 20 words to the original instruction.\n",
    "    ### Step 5: Review the rewritten instruction and check for any inaccuracies or inconsistencies. Make sure that the rewritten instruction is a more complex version of the original instruction and not a completely different problem. If any parts of the rewritten instruction are unreasonable or do not fit the problem context, revise them as necessary.\n",
    "\n",
    "    Please reply strictly in the following format:\n",
    "    \n",
    "    ### Step 1 #Elements Identified#:\n",
    "    ### Step 2 #Complexity Additions#:\n",
    "    ### Step 3 #Plan#:\n",
    "    ### Step 4 #Rewritten Instruction#:\n",
    "    ### Step 5 #Finally Rewritten Instruction#:\n",
    "\n",
    "    #Instruction#: {instruction}\n",
    "    \"\"\"\n",
    "    output = call_llm(prompt, **kwargs)\n",
    "    rewritten_instruction = re.search(r\"Step 5 #Finally Rewritten Instruction#:(.*)\", output, re.DOTALL).group(1).strip()\n",
    "    return rewritten_instruction\n",
    "\n",
    "def get_step_2_evolved_instruction(instruction, **kwargs):\n",
    "    prompt = f\"\"\"You are an Instruction Rewriter that rewrites the given #Instruction# into a more complex version. Please follow the steps below to rewrite the given \"#Instruction#\" into a more complex version.\n",
    "\n",
    "    ### Step 1: Carefully read the initial instruction and identify all the elements involved - this includes variables, constants, operations, and conditions.\n",
    "    ### Step 2: Consider how each element could be made more complex. For variables, this could involve introducing more variables or making the existing variables dependent on others. For constants, consider changing them to variables or making them dependent on other factors. For operations, consider introducing more complex operations or multiple steps. For conditions, consider adding more conditions or making the existing conditions more complex.\n",
    "    ### Step 3: Formulate a plan to integrate these complexities into the instruction. Ensure that the changes are coherent and relevant to the initial problem context. The plan should not just randomly add complexity but should make the problem more interesting or challenging in a meaningful way. Avoid introducing irrelevant concepts or complicating the problem to the extent of changing its nature.\n",
    "    ### Step 4: Rewrite the instruction according to the plan. Ensure that the rewritten instruction is still understandable and that it accurately represents the initial problem context. The rewritten instruction should only add 10 to 20 words to the original instruction. Make sure that the progression of complexity is smooth and gradual.\n",
    "    ### Step 5: Review the rewritten instruction and check for any inaccuracies or inconsistencies. Make sure that the rewritten instruction is a more complex version of the original instruction and not a completely different problem. If any parts of the rewritten instruction are unreasonable or do not fit the problem context, revise them as necessary.\n",
    "\n",
    "    Please reply strictly in the following format:\n",
    "    \n",
    "    ### Step 1 #Elements Identified#:\n",
    "    ### Step 2 #Complexity Additions#:\n",
    "    ### Step 3 #Plan#:\n",
    "    ### Step 4 #Rewritten Instruction#:\n",
    "    ### Step 5 #Finally Rewritten Instruction#:\n",
    "\n",
    "    #Instruction#: {instruction}\n",
    "    \"\"\"\n",
    "    output = call_llm(prompt, **kwargs)\n",
    "    rewritten_instruction = re.search(r\"Step 5 #Finally Rewritten Instruction#:(.*)\", output, re.DOTALL).group(1).strip()\n",
    "    return rewritten_instruction\n",
    "\n",
    "def get_step_k_evolved_instruction(instruction, **kwargs):\n",
    "    prompt = f\"\"\"You are an Instruction Rewriter that rewrites the given #Instruction# into a more complex version. Please follow the steps below to rewrite the given \"#Instruction#\" into a more complex version.\n",
    "\n",
    "    ### Step 1: Carefully read the initial instruction and identify all the elements involved - this includes variables, constants, operations, and conditions.\n",
    "    ### Step 2: Consider how each element could be made more complex. For variables, this could involve introducing more variables or making the existing variables dependent on others. For constants, consider changing them to variables or making them dependent on other factors. For operations, consider introducing more complex operations or multiple steps. For conditions, consider adding more conditions or making the existing conditions more complex.\n",
    "    ### Step 3: Formulate a plan to integrate these complexities into the instruction. Ensure that the changes are coherent and relevant to the initial problem context. The plan should not just randomly add complexity but should make the problem more interesting or challenging in a meaningful way. Avoid introducing irrelevant concepts or complicating the problem to the extent of changing its nature.\n",
    "    ### Step 4: Rewrite the instruction according to the plan. Ensure that the rewritten instruction is still understandable and that it accurately represents the initial problem context. The rewritten instruction should only add 10 to 20 words to the original instruction. Make sure that the progression of complexity is smooth and gradual.\n",
    "    ### Step 5: Review the rewritten instruction and check for any inaccuracies or inconsistencies. Make sure that the rewritten instruction is a more complex version of the original instruction and not a completely different problem. If any parts of the rewritten instruction are unreasonable or do not fit the problem context, revise them as necessary.\n",
    "    ### Step 6: Ensure that the complexity increase is consistent and logical. Avoid introducing new conditions or variables that are not related to the initial problem. The complexity should evolve from the initial problem and not transform it into a different problem.\n",
    "    ### Step 7: Test the rewritten instruction to ensure that it is solvable and that the complexity has indeed increased. If the problem is too difficult or impossible to solve, revise it as necessary.\n",
    "\n",
    "    Please reply strictly in the following format:\n",
    "    \n",
    "    ### Step 1 #Elements Identified#:\n",
    "    ### Step 2 #Complexity Additions#:\n",
    "    ### Step 3 #Plan#:\n",
    "    ### Step 4 #Rewritten Instruction#:\n",
    "    ### Step 5 #Revisied Instruction#:\n",
    "    ### Step 6 #Consistency Check#:\n",
    "    ### Step 7 #Finally Rewritten Instruction#:\n",
    "\n",
    "    #Instruction#: {instruction}\n",
    "    \"\"\"\n",
    "    output = call_llm(prompt, **kwargs)\n",
    "    rewritten_instruction = re.search(r\"Step 7 #Finally Rewritten Instruction#:(.*)\", output, re.DOTALL).group(1).strip()\n",
    "    return rewritten_instruction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "FEEDBACK_ASPECTS = [\n",
    "    (\"Complexity\", \"Assess whether the complexity of instructions has increased sufficiently. See if more details and steps have been added.\"),\n",
    "    (\"Diversity\", \"Evaluate whether new scenarios, examples, or contexts are introduced. Make sure they cover a variety of situations.\"),\n",
    "    (\"Preservation of key information:\", \"Evaluate whether key information has been maintained. Make sure important concepts and facts are not missing.\"),\n",
    "    # (\"Clarity\", \"Assess whether instructions are clearly communicated. Make sure they are easy to understand and unambiguous.\"),\n",
    "    # (\"Misunderstanding\", \"Evaluate instructions to ensure they are not misleading. Identify areas that may be misinterpreted.\"),\n",
    "    # (\"Different contexts\", \"Assess whether the instructions can be applied to a variety of situations and contexts. Make sure they are generalizable.\")\n",
    "]\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "def generate_feedback(original_instruction, evolved_instruction, feedback_aspect, top_p=0.95, temperature=0.5):\n",
    "    feedback_prompt = f\"\"\"    \n",
    "    The following list shows cases where an #Instruction# evolves into a more complex version of an Instruction, #Evolved Instruction#.\n",
    "    Analyze the evolution of the #Instruction# and provide detailed feedback on the following aspects:\n",
    "\n",
    "    Provide concise feedback on the #Aspect# in under 100 characters:\n",
    "    #Aspect#: {feedback_aspect}#:\n",
    "\n",
    "    #Instruction#: {original_instruction}\n",
    "    #Evolved Instruction#: {evolved_instruction}\n",
    "\n",
    "    Please answer concisely in the format below and provide a reason for each item within 100 characters using the format examples below. Please provide only one feedback.\n",
    "    \n",
    "    If feedback is required, please write the #Need Feedback#, #Issue Name#, #Reason#, and #Feedback# using the format below.\n",
    "    \n",
    "    #Need Feedback#: \n",
    "    #Issue Name#:\n",
    "    #Reason#:\n",
    "    #Feedback#:\n",
    "\n",
    "    If feedback is not required, please write the #Need Feedback# using the format below.\n",
    "    \n",
    "    #Need Feedback#: No\n",
    "    \n",
    "    Please refer to the example below for an accurate answer.\n",
    "\n",
    "    [Example 1]\n",
    "    #Need Feedback#: Yes\n",
    "    #Issue Name#: Lack of consistency and logical progression in complexity\n",
    "    #Reason#: Introduce new concepts or variables without building on the previous instruction, making the problem confusing or unrealistic.\n",
    "    #Feedback#: Build on previous instructions to improve consistency and transform into logical sentences\n",
    "\n",
    "    [Example 2]\n",
    "    #Need Feedback#: Yes\n",
    "    #Issue Name#: Incorrect or unrealistic mathematical calculations\n",
    "    #Reason#: Introduce mathematical operations or equations that don’t make sense in the context of the problem or are mathematically incorrect.\n",
    "    #Feedback#: Make accurate mathematical calculations. Take advantage of think step-by-step.\n",
    "\n",
    "    [Example 3]\n",
    "    #Need Feedback#: Yes\n",
    "    #Issue Name#: Inconsistent or contradictory information\n",
    "    #Issue Description#: Introduce information that contradicts previous instruction.\n",
    "    #Feedback#: Ensure consistency in the information provided. Avoid introducing contradictory information.\n",
    "\n",
    "    [Example 4]\n",
    "    #Need Feedback#: No\n",
    "    \"\"\"\n",
    "    return call_llm(feedback_prompt, top_p=top_p, temperature=temperature)\n",
    "\n",
    "def convert_feedback_needed(feedback_list):\n",
    "    return [True if '#Need Feedback#: Yes' in feedback else False for feedback in feedback_list]\n",
    "\n",
    "def generate_feedback_parallel(original_instruction, evolved_instruction, m=1):\n",
    "    feedback_list = []\n",
    "    with ThreadPoolExecutor(max_workers=m) as executor:\n",
    "        futures = [\n",
    "            executor.submit(\n",
    "                generate_feedback,\n",
    "                original_instruction,\n",
    "                evolved_instruction,\n",
    "                random.choice(FEEDBACK_ASPECTS),\n",
    "                random.uniform(0.8, 1.0),  # top_p range\n",
    "                random.uniform(0.6, 0.9)   # temperature range\n",
    "            ) for _ in range(m)\n",
    "        ]\n",
    "        for future in futures:\n",
    "            feedback_list.append(future.result())\n",
    "    return convert_feedback_needed(feedback_list), feedback_list\n",
    "\n",
    "def evaluate_feedback(feedback_list):\n",
    "    failure_count = 0\n",
    "    for feedback in feedback_list:\n",
    "        if \"#Need Feedback#: Yes\" in feedback:\n",
    "            failure_count += 1\n",
    "\n",
    "    failure_rate = failure_count / len(feedback_list)\n",
    "    return failure_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_instruction = \"Explain the concept of gravity.\"\n",
    "evolved_instruction = get_step_0_evolved_instruction(original_instruction, max_tokens=512)\n",
    "feedback_required, feedback_list = generate_feedback_parallel(original_instruction, evolved_instruction, m=1)\n",
    "print(feedback_required, feedback_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_optimized_instruction(combined_feedback, current_instruction, **kwargs):\n",
    "    if combined_feedback != \"\":\n",
    "        optimize_prompt = f\"\"\"\n",
    "        #Current Instruction#: {current_instruction}\n",
    "        #Feedback#: {combined_feedback}\n",
    "\n",
    "        I will provide you with the method for evolving the above #Current Instructions#.\n",
    "        You need to optimize this method based on the #Feedback# from the evolution failure case, without harming the performance on other cases, and ensure that the complexity increase brought by the optimized method is not lower than the previous method.\n",
    "        Please provide the optimized method in the following format. \n",
    "\n",
    "        #Optimized Instruction#:   \n",
    "        \"\"\"\n",
    "        \n",
    "        output = call_llm(optimize_prompt, **kwargs)\n",
    "        rewritten_instruction = re.search(r\"#Optimized Instruction#:(.*)\", output, re.DOTALL).group(1).strip()\n",
    "\n",
    "        return rewritten_instruction\n",
    "    else:\n",
    "        return current_instruction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_failure_rate(feedback_list):\n",
    "    failure_count = 0\n",
    "    for feedback in feedback_list:\n",
    "        if \"#Need Feedback#: Yes\" in feedback:\n",
    "            failure_count += 1\n",
    "\n",
    "    failure_rate = failure_count / len(feedback_list)\n",
    "    return failure_rate\n",
    "\n",
    "def get_feeback_text(feedback_required, feedback_list):\n",
    "    import re\n",
    "    feedback_pattern = r\"#Feedback#:\\s*(.*)\"\n",
    "    final_feedback_text = \"\"\n",
    "\n",
    "    for do_feedback, feedback in zip(feedback_required, feedback_list):\n",
    "        if do_feedback:\n",
    "            # Extracting #Feedback# using regular expressions\n",
    "            feedback_match = re.search(feedback_pattern, feedback)\n",
    "\n",
    "            if feedback_match:\n",
    "                feedback_text = feedback_match.group(1)\n",
    "                final_feedback_text += f\"\\n{feedback_text}\"\n",
    "    return final_feedback_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_instruction = \"Explain the concept of gravity.\"\n",
    "evolved_instruction = get_step_0_evolved_instruction(original_instruction, max_tokens=512)\n",
    "feedback_required, feedback_list = generate_feedback_parallel(original_instruction, evolved_instruction, m=1)\n",
    "print(feedback_required, feedback_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimized_instruction = get_optimized_instruction(get_feeback_text(feedback_required, feedback_list), evolved_instruction)a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evolved_instruction = get_step_1_evolved_instruction(optimized_instruction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feedback_required, feedback_list = generate_feedback_parallel(optimized_instruction, evolved_instruction, m)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py312-dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}