quick_start/08_Prompt_engineering_few_shot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from openai import AzureOpenAI\n",
    "import tiktoken\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "client = AzureOpenAI(\n",
    "    azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\",\"\").strip(),\n",
    "    api_key        = os.getenv(\"AZURE_OPENAI_API_KEY\"),\n",
    "    api_version    = os.getenv(\"OPENAI_API_VERSION\")\n",
    ")\n",
    "\n",
    "encoding = tiktoken.encoding_for_model(\"gpt-3.5-turbo\")\n",
    "deployment_name = os.getenv('DEPLOYMENT_NAME')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A Few Shot Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "😍,👏,👍\n"
     ]
    }
   ],
   "source": [
    "# Zero-shot classification\n",
    "system_prompt =\"\"\"Predict up to 5 emojis as a response to a text chat message. The output\n",
    "should only include emojis.\n",
    "\n",
    "input: The new visual design is blowing my mind 🤯\n",
    "output: ➕,💘, ❤‍🔥\n",
    "\n",
    "input: Well that looks great regardless\n",
    "output: ❤️,🪄\n",
    "\n",
    "input: Unfortunately this won't work\n",
    "output: 💔,😔\n",
    "\n",
    "input: sounds good, I'll look into that\n",
    "output: 🙏,👍\n",
    "\n",
    "input: 10hr cut of jeff goldblum laughing URL\n",
    "output: 😂,💀,⚰️\n",
    "\"\"\"\n",
    "user_prompt = \"The new user interface is amazing!\"\n",
    "response = client.chat.completions.create(\n",
    "    model=deployment_name,\n",
    "    messages = [{\"role\":\"system\", \"content\":system_prompt},\n",
    "                {\"role\":\"user\",\"content\": user_prompt,}])\n",
    "print(response.choices[0].message.content)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chain of Thought\n",
    "\n",
    "Experiment results demonstrate Zero-shot-CoT using single prompt template, significantly outperform zero-shot LLM performance on diverse benchmark reasoning tasks.  Without any hand-crafted few-shot examples, e.g. increasing the accuracy on MultiArith from 17.7% to 78.7% and GSM8K from 10.4% to 40.7% with large InstructGPT model (text-davinci-002).\n",
    "\n",
    "Source: [Large Language Models are Zero-Shot Reasoners](https://arxiv.org/abs/2205.11916)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def call_openai(prompt, temperature=1, max_token=60):\n",
    "    response = client.chat.completions.create(\n",
    "        model=deployment_name,\n",
    "        messages = prompt,\n",
    "            max_tokens=max_token,\n",
    "            temperature = temperature\n",
    "    )\n",
    "    return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "To determine how many loaves of bread the Beverly Hills Bakery had left, we need to account for the loaves sold and the loaves returned.\n",
      "\n",
      "1. Start with the total number of loaves baked:\n",
      "   \\[\n",
      "   200 \\text{ loaves}\n",
      "   \\]\n",
      "\n",
      "2. Subtract the number of loaves sold in the morning:\n",
      "   \\[\n",
      "   200 - 93 = 107 \\text{ loaves}\n",
      "   \\]\n",
      "\n",
      "3. Subtract the number of loaves sold in the afternoon:\n",
      "   \\[\n",
      "   107 - 39 = 68 \\text{ loaves}\n",
      "   \\]\n",
      "\n",
      "4. Add the number of loaves returned by the grocery store:\n",
      "   \\[\n",
      "   68 + 6 = 74 \\text{ loaves}\n",
      "   \\]\n",
      "\n",
      "Therefore, the bakery had 74 loaves of bread left.\n"
     ]
    }
   ],
   "source": [
    "# This prompt gets wrong answer - ZERO_SHOT Prompt\n",
    "\n",
    "system_prompt = \"\"\"Answer the following questions:\"\"\"\n",
    "quiz = \"\"\"The bakers at the Beverly Hills Bakery baked 200 loaves of bread on Monday morning.\n",
    "They sold 93 loaves in the morning and 39 loaves in the afternoon.\n",
    "A grocery store returned 6 unsold loaves.\n",
    "How many loaves of bread did they have left?\"\"\"\n",
    "\n",
    "prompt = [\n",
    "    {\"role\": \"system\", \"content\": system_prompt},\n",
    "    {\"role\": \"user\", \"content\": quiz}\n",
    "]\n",
    "\n",
    "response_msg = call_openai(prompt, temperature=0, max_token=200)\n",
    "\n",
    "print(response_msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "To find out how many loaves of bread the bakers at the Beverly Hills Bakery had left, we need to follow these steps:\n",
      "\n",
      "1. Start with the total number of loaves baked on Monday morning: 200 loaves.\n",
      "2. Subtract the number of loaves sold in the morning: 200 - 93 = 107 loaves.\n",
      "3. Subtract the number of loaves sold in the afternoon: 107 - 39 = 68 loaves.\n",
      "4. Add the number of loaves returned by the grocery store: 68 + 6 = 74 loaves.\n",
      "\n",
      "So, the bakers had 74 loaves of bread left.\n"
     ]
    }
   ],
   "source": [
    "# Still wrong answer with few-shot learning\n",
    "\n",
    "prompt=[\n",
    "    {\"role\": \"system\", \"content\": system_prompt},\n",
    "    {\"role\": \"user\", \"content\": \"Q: Roger has 5 tennis balss. He buys 2 more cans of tennis balls. Each can has 3 tennis balls. How many tennis balls does Roger have now?\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"A: The answer is 11.\"},\n",
    "    {\"role\": \"user\", \"content\": quiz}\n",
    "]\n",
    "\n",
    "response_msg = call_openai(prompt, temperature=0, max_token=200)\n",
    "\n",
    "print(response_msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let's break down the problem step by step to find out how many loaves of bread the Beverly Hills Bakery had left.\n",
      "\n",
      "1. **Initial number of loaves baked:**\n",
      "   - The bakers baked 200 loaves of bread on Monday morning.\n",
      "\n",
      "2. **Number of loaves sold in the morning:**\n",
      "   - They sold 93 loaves in the morning.\n",
      "\n",
      "3. **Number of loaves sold in the afternoon:**\n",
      "   - They sold 39 loaves in the afternoon.\n",
      "\n",
      "4. **Number of loaves returned by the grocery store:**\n",
      "   - A grocery store returned 6 unsold loaves.\n",
      "\n",
      "First, let's calculate the total number of loaves sold during the day:\n",
      "\\[ 93 \\text{ (morning sales)} + 39 \\text{ (afternoon sales)} = 132 \\text{ loaves sold} \\]\n",
      "\n",
      "Next, we subtract the total number of loaves sold from the initial number of loaves baked to find out how\n"
     ]
    }
   ],
   "source": [
    "# With CoT, the answer is correct\n",
    "\n",
    "cot_message = \" Understand the sentences well and let’s think step by step.\"\n",
    "\n",
    "prompt=[\n",
    "    {\"role\": \"system\", \"content\": system_prompt},\n",
    "    {\"role\": \"user\", \"content\": quiz + cot_message}\n",
    "]\n",
    "\n",
    "response_msg = call_openai(prompt, temperature=0, max_token=200)\n",
    "\n",
    "print(response_msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let's break down the problem step by step:\n",
      "\n",
      "1. The bakers baked 200 loaves of bread on Monday morning.\n",
      "2. They sold 93 loaves in the morning.\n",
      "3. They sold 39 loaves in the afternoon.\n",
      "\n",
      "First, let's calculate the total number of loaves sold:\n",
      "\\[ 93 \\text{ (morning sales)} + 39 \\text{ (afternoon sales)} = 132 \\text{ loaves sold} \\]\n",
      "\n",
      "Next, let's determine how many loaves were left after the sales:\n",
      "\\[ 200 \\text{ (initial loaves)} - 132 \\text{ (loaves sold)} = 68 \\text{ loaves remaining} \\]\n",
      "\n",
      "Then, a grocery store returned 6 unsold loaves:\n",
      "\\[ 68 \\text{ (remaining loaves)} + 6 \\text{ (returned loaves)} = 74 \\text{ loaves left} \\]\n",
      "\n",
      "So, the bakers had 74 loaves\n"
     ]
    }
   ],
   "source": [
    "# PROMPT_FEW_SHOT_CoT\n",
    "\n",
    "prompt=[\n",
    "    {\"role\": \"system\", \"content\": system_prompt},\n",
    "    {\"role\": \"user\", \"content\": \"\"\"Roger has 5 tennis balls. He buys 2 more cans of tennis balls. Each can has 3 tennis balls. How many tennis balls does he have now?\"\"\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"\"\"Roger started with 5 balls. 2 cans of 3 tennis balls each is 6 tennis balls. 5 + 6 = 11. The answer is 11.\"\"\"},\n",
    "    {\"role\": \"user\", \"content\": quiz + cot_message}\n",
    "]\n",
    "\n",
    "response_msg = call_openai(prompt, temperature=0, max_token=200)\n",
    "\n",
    "print(response_msg)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Commonsense Reasoning\n",
    "\n",
    "Paper: [Generated Knowledge Prompting for Commonsense Reasoning](https://arxiv.org/abs/2110.08387)\n",
    "\n",
    "\n",
    "\n",
    "Provide knowledge, turn knowledge question into reasoning. In general, more knowledge, better result.\n",
    "\n",
    "3 Contributing factors:\n",
    "\n",
    "(i) the quality of knowledge, \n",
    "\n",
    "(ii) the quantity of knowledge where the performance improves with more knowledge statements, and \n",
    "\n",
    "(iii) the strategy for integrating knowledge during inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False. In golf, the objective is to have the lowest score possible. Players aim to complete the course in the fewest number of strokes.\n"
     ]
    }
   ],
   "source": [
    "common_question = \"\"\"The player with the lowest score wins.\n",
    "Is this true or false: Part of golf is trying to get a higher point total than others.\n",
    "\"\"\"\n",
    "\n",
    "prompt = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "    {\"role\": \"user\", \"content\": common_question},\n",
    "]\n",
    "\n",
    "response_msg = call_openai(prompt, temperature=0, max_token=100)\n",
    "\n",
    "print(response_msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An easel typically has three legs, similar to a tripod. However, some easels, especially those designed for larger canvases or more stability, may have four legs. The traditional artist's easel usually has three legs, making it similar to a tripod in terms of structure.\n"
     ]
    }
   ],
   "source": [
    "common_question = \"\"\"A tripod is a kind of easel\n",
    "How many legs does an easel have?\n",
    "\"\"\"\n",
    "prompt = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "    {\"role\": \"user\", \"content\": common_question},\n",
    "]\n",
    "\n",
    "response_msg = call_openai(prompt, temperature=0, max_token=100)\n",
    "\n",
    "print(response_msg)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check out follow 2 examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No, the statement is incorrect. The objective of golf is not to get a higher point total than others. Instead, the goal is to complete a set of holes, typically 18, in the fewest number of strokes possible. Each stroke counts as one point, and the player with the lowest total number of strokes at the end of the round is the winner. Therefore, in golf, a lower score is better.\n"
     ]
    }
   ],
   "source": [
    "# High confidence answer\n",
    "common_question = \"\"\"Question: Part of golf is trying to get a higher point total than others. Yes or No?\n",
    "Knowledge: The objective of golf is to play a set of holes in the least number of strokes. A round of golf typically consists of 18 holes. Each hole is played once in the round on a standard golf course. Each stroke is counted as one point, and the total number of strokes is used to determine the winner of the game.\n",
    "Explain and Answer: \n",
    "\"\"\"\n",
    "prompt=[\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "    {\"role\": \"user\", \"content\": common_question},\n",
    "]\n",
    "\n",
    "response_msg = call_openai(prompt, temperature=0, max_token=100)\n",
    "\n",
    "print(response_msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No, part of golf is not trying to get a higher point total than others. \n",
      "\n",
      "In golf, the objective is to achieve the lowest score possible. Players use various types of clubs to hit a ball into a series of holes on a course, and the score is calculated by adding up the total number of strokes taken to complete each hole. The player with the fewest strokes, and thus the lowest score, wins the game. Therefore, unlike many other sports where a higher score indicates better performance\n"
     ]
    }
   ],
   "source": [
    "# Low confidence answer. 예문을 더 애매하게 바꿔야... 잘못된 결과가 나올것 같은데... 정상으로 나옴.\n",
    "common_question = \"\"\"Question: Part of golf is trying to get a higher point total than others. Yes or No?\n",
    "Knowledge: Golf is a precision club-and-ball sport in which competing players (or golfers) use several types of clubs to hit a ball into a series of holes on a course using the fewest number of strokes. A goal is highest rated with the lowest score calculated by adding up the total number of strokes taken on each hole. The player with the lowest score wins the game.\n",
    "Explain and Answer: \n",
    "\"\"\"\n",
    "prompt=[\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "    {\"role\": \"user\", \"content\": common_question},\n",
    "]\n",
    "\n",
    "response_msg = call_openai(prompt, temperature=0, max_token=100)\n",
    "\n",
    "print(response_msg)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "azureml_py38",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6d65a8c07f5b6469e0fc613f182488c0dccce05038bbda39e5ac9075c0454d11"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}