From de8a1f04aede45118ff9bb1d4948aac82ec948b8 Mon Sep 17 00:00:00 2001 From: Shahul ES Date: Tue, 21 Nov 2023 10:24:53 +0530 Subject: [PATCH] Convert context_recall to json format (#312) --- .../assesments/Deterministic output.ipynb | 583 ++++++++++++++++++ src/ragas/metrics/_context_recall.py | 84 ++- src/ragas/metrics/base.py | 2 +- src/ragas/validation.py | 1 + 4 files changed, 649 insertions(+), 21 deletions(-) create mode 100644 experiments/assesments/Deterministic output.ipynb diff --git a/experiments/assesments/Deterministic output.ipynb b/experiments/assesments/Deterministic output.ipynb new file mode 100644 index 000000000..b3653d472 --- /dev/null +++ b/experiments/assesments/Deterministic output.ipynb @@ -0,0 +1,583 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "ed913729", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/ragas/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import os\n", + "import openai\n", + "\n", + "from openai import OpenAI\n", + "from datasets import load_dataset\n", + "import json\n", + "import numpy as np\n", + "from scipy.stats import entropy\n", + "import matplotlib.pyplot as plt\n", + "client = OpenAI()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b2de3222", + "metadata": {}, + "outputs": [], + "source": [ + "def llm2(prompt, **kwargs):\n", + " response = client.chat.completions.create(\n", + " model=kwargs.get(\"model\", \"gpt-3.5-turbo\"),\n", + " messages=[{\"role\": \"system\", \"content\": prompt}],\n", + " temperature=kwargs.get(\"temperature\", 0),\n", + " top_p=kwargs.get(\"top_p\", 1),\n", + " frequency_penalty=kwargs.get(\"frequency_penalty\", 0.0),\n", + " presence_penalty=kwargs.get(\"presence_penalty\", 0.0),\n", + " max_tokens=kwargs.get(\"max_tokens\", 500),\n", + " n=kwargs.get(\"n\", 1),\n", + " )\n", + " return response" + ] + }, + { + "cell_type": "markdown", + "id": "c46a9958", + "metadata": {}, + "source": [ + "## Experiment 1\n", + "AIM: To quantify the consistency of ragas recall score \n", + "\n", + "- before prompt change\n", + "- After prompt change\n", + "- Prompt change + self consistency method\n", + "\n", + "Quantify \n", + "- KL divergance\n", + "- Mean absolute change (x_i - y_i)\n", + "- Mean of scores\n", + "\n", + "Dataset used:\n", + "- explodinggradients/WikiEval" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ecf10292", + "metadata": {}, + "outputs": [], + "source": [ + "# with open(\"consistency_recall.json\",'w') as file:\n", + "# json.dump({}, file, indent=4)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "34acc81f", + "metadata": {}, + "outputs": [], + "source": [ + "def read_score(name):\n", + " filename = \"consistency_recall.json\"\n", + " data = json.load(open(filename))\n", + " return data.get(name)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ba69e301", + "metadata": {}, + "outputs": [], + "source": [ + "def record_score(score,name):\n", + " \n", + " filename = \"consistency_recall.json\"\n", + " data = json.load(open(filename))\n", + " data[name] = score\n", + " with open(filename,'w') as file:\n", + " json.dump(data, file, indent=4)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cf059b7b", + "metadata": {}, + "outputs": [], + "source": [ + "def KL(dist1,dist2):\n", + " \n", + " if isinstance(dist1, list):\n", + " dist1 = np.array(dist1) + 1e-8\n", + " \n", + " if isinstance(dist2, list):\n", + " dist2 = np.array(dist2) + 1e-8\n", + " \n", + " dist1_normalized = dist1 / dist1.sum()\n", + " dist2_normalized = dist2 / dist2.sum()\n", + " kl_divergence = entropy(dist1_normalized, dist2_normalized)\n", + " return kl_divergence" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "id": "ae5d00fc", + "metadata": {}, + "outputs": [], + "source": [ + "recall_prompt = \"\"\"\n", + "Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Output json with reason.\n", + "\n", + "\n", + "question: What can you tell me about albert Albert Einstein?\n", + "context: Albert Einstein (14 March 1879 – 18 April 1955) was a German-born theoretical physicist,widely held to be one of the greatest and most influential scientists of all time. Best known for developing the theory of relativity, he also made important contributions to quantum mechanics, and was thus a central figure in the revolutionary reshaping of the scientific understanding of nature that modern physics accomplished in the first decades of the twentieth century. His mass–energy equivalence formula E = mc2, which arises from relativity theory, has been called \"the world's most famous equation\". He received the 1921 Nobel Prize in Physics \"for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect\", a pivotal step in the development of quantum theory. His work is also known for its influence on the philosophy of science. In a 1999 poll of 130 leading physicists worldwide by the British journal Physics World, Einstein was ranked the greatest physicist of all time. His intellectual achievements and originality have made Einstein synonymous with genius.\n", + "answer: Albert Einstein born in 14 March 1879 was German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. He received the 1921 Nobel Prize in Physics \"for his services to theoretical physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895 \n", + "classification:\n", + "[\n", + " {{ \"statement_1\":\"Albert Einstein, born on 14 March 1879, was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time.\",\n", + " \"reason\": \"The date of birth of Einstein is mentioned clearly in the context.\",\n", + " \"Attributed\": \"Yes\"\n", + " }},\n", + " {{\n", + " \"statement_2\":\"He received the 1921 Nobel Prize in Physics 'for his services to theoretical physics.\",\n", + " \"reason\": \"The exact sentence is present in the given context.\",\n", + " \"Attributed\": \"Yes\"\n", + " }},\n", + " {{\n", + " \"statement_3\": \"He published 4 papers in 1905.\",\n", + " \"reason\": \"There is no mention about papers he wrote in the given context.\",\n", + " \"Attributed\": \"No\"\n", + " }},\n", + " {{\n", + " \"statement_4\":\"Einstein moved to Switzerland in 1895.\",\n", + " \"reason\": \"There is no supporting evidence for this in the given context.\",\n", + " \"Attributed\": \"No\"\n", + " }}\n", + "]\n", + "\n", + "question: who won 2020 icc world cup?\n", + "context: Who won the 2022 ICC Men's T20 World Cup?\n", + "The 2022 ICC Men's T20 World Cup, held from October 16 to November 13, 2022, in Australia, was the eighth edition of the tournament. Originally scheduled for 2020, it was postponed due to the COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets in the final to clinch their second ICC Men's T20 World Cup title.\n", + "answer: England \n", + "classification: \n", + "[\n", + " {{\n", + " \"statement_1\":\"England won the 2022 ICC Men's T20 World Cup.\",\n", + " \"reason\": \"From context it is clear that England defeated Pakistan to win the World Cup.\",\n", + " \"Attributed\": \"Yes\"\n", + " }}\n", + "]\n", + "\n", + "question:{question}\n", + "context:{context}\n", + "answer:{answer}\n", + "classification:\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "id": "757d9b75", + "metadata": {}, + "outputs": [], + "source": [ + "c = \"\"\"\n", + "Black holes, one of the most enigmatic and fascinating phenomena in the universe, are regions of spacetime where gravity is so intense that nothing, not even light, can escape their grasp. Formed from the remnants of massive stars that have undergone gravitational collapse, black holes are characterized by their event horizon, a boundary beyond which all matter and radiation are irretrievably pulled in. Intriguingly, while they are invisible, their presence can be inferred through their interaction with nearby matter and the emission of high-energy radiation. Black holes challenge our understanding of physics, particularly at the intersection of quantum mechanics and general relativity, presenting intriguing questions about the nature of space, time, and the fabric of the universe itself.\n", + "\"\"\"\n", + "q = \"What is the term used to describe the boundary around a black hole where nothing, not even light, can escape?\"\n", + "a = \"Event Horizon.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "id": "6fc3ebd7", + "metadata": {}, + "outputs": [], + "source": [ + "output = llm2(recall_prompt.format(context=c,question=q,answer=a))" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "id": "46b0fca3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'statement_1': 'The term used to describe the boundary around a black hole where nothing, not even light, can escape is the event horizon.',\n", + " 'reason': \"The exact term 'event horizon' is mentioned in the context.\",\n", + " 'Attributed': 'Yes'}]" + ] + }, + "execution_count": 103, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval(output.choices[0].message.content)" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "id": "72bdeac3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'[\\n {\\n \"statement_1\":\"The term used to describe the boundary around a black hole where nothing, not even light, can escape is the event horizon.\",\\n \"reason\": \"The exact term \\'event horizon\\' is mentioned in the context.\",\\n \"Attributed\": \"Yes\"\\n }\\n]'" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output.choices[0].message.content" + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "id": "3485a0fb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'statement_1': 'The term used to describe the boundary around a black hole where nothing, not even light, can escape is the event horizon.',\n", + " 'reason': \"The exact term 'event horizon' is mentioned in the context.\",\n", + " 'Attributed': 'Yes'}]" + ] + }, + "execution_count": 121, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "eval(match.group())" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7eaa3091", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset parquet (/Users/shahules/.cache/huggingface/datasets/Pakulski___parquet/Pakulski--ELI5-test-ed159b4d22db0b30/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)\n", + "100%|█████████████████████████████████████████████████████| 1/1 [00:00<00:00, 7.06it/s]\n" + ] + } + ], + "source": [ + "# dataset = load_dataset(\"explodinggradients/WikiEval\")\n", + "dataset = load_dataset(\"Pakulski/ELI5-test\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "bfd13691", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.metrics import context_recall\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b1b1fc2a", + "metadata": {}, + "outputs": [], + "source": [ + "# from ragas.metrics._context_recall import ContextRecallImproved\n", + "# context_recall = ContextRecallImproved()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "dc8c4899", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas import evaluate" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "83f0a0de", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "dataset = dataset['train'].select(range(0,10))\n", + "dataset = dataset.rename_columns({\"document\":\"contexts\",\"goldenAnswer\":\"ground_truths\"})\n", + "dataset = dataset.map(lambda x : {'ground_truths':[x['ground_truths']],'contexts':[x['contexts']] })" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "bc5daf8e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading cached processed dataset at /Users/shahules/.cache/huggingface/datasets/explodinggradients___parquet/explodinggradients--WikiEval-33bd2cbc490cc57b/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7/cache-5fe04aab11621603.arrow\n" + ] + } + ], + "source": [ + "# dataset = dataset['train'].map(lambda x : {'ground_truths':[x['ungrounded_answer']]})" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "33ff7b32", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Dataset({\n", + " features: ['id', 'question', 'contexts', 'ground_truths'],\n", + " num_rows: 100\n", + "})" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "80917afa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "evaluating with [context_recall]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████| 4/4 [01:13<00:00, 18.34s/it]\n" + ] + } + ], + "source": [ + "# ragas_score = evaluate(dataset=dataset,metrics=[context_recall],column_map={\"question\":\"question\",\"contexts\":\"context_v1\",\"ground_truths\":\"ground_truths\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b29ec575", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "evaluating with [context_recall]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████| 1/1 [00:23<00:00, 23.22s/it]\n" + ] + } + ], + "source": [ + "ragas_score = evaluate(dataset=dataset,metrics=[context_recall])" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "aaa176e8", + "metadata": {}, + "outputs": [], + "source": [ + "record_score(ragas_score.to_pandas()['context_recall'].values.tolist(),'prompt_to_json_eli5_1')" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "5642abde", + "metadata": {}, + "outputs": [], + "source": [ + "score = KL(read_score(\"prompt_to_json_eli5\"),read_score(\"prompt_to_json_eli5_1\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "25333eb5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 66, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "score" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "982131d0", + "metadata": {}, + "outputs": [], + "source": [ + "record_score(score,\"prompt_to_json_prompt_to_json_2\")" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "id": "de059ca8", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAFRCAYAAADaTrE/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAAsTAAALEwEAmpwYAAAzdUlEQVR4nO3deXhTdaLG8TdJW9pSKDQUapUBLSCLXBSrCDgsQ3XuwFWQEVFBRWQcRRRwFBFZFOEBF0QEHBGQRXmkbizX3cJlEZRhq+LKUjdsC11YS2uXc+4flUhogQSac5L2+3keHsjvnCRvkvLj5WxxmKZpCgAAALZx2h0AAACgpqOQAQAA2IxCBgAAYDMKGQAAgM0oZAAAADajkAEAANiMQgYAgA0GDRqklJQUu2MgSFDIYIsTJyLDMPTss8/qkksuUe3atVWvXj21a9dOY8eO9bpPSUmJnn76af3Xf/2XoqKiVLduXXXp0kXvvPOO13o//vijHA6H6tSpo3379nktGzJkiLp16xbQ1wagehg0aJAcDodGjRrlNb537145HA6tWbPGp8d57bXX5HA4KozPmDFDb775ZlVERTVAIYPtJk6cqMmTJ2v06NH68ssvtWHDBo0ZM0YFBQWedUpKSvS3v/1N06ZN04gRI/TNN9/o888/V48ePdS/f389/vjjFR63tLRUEyZMsPCVAKhuIiMj9cILL+inn36q8seOjY1V/fr1q/xxEZooZLDd8uXLddddd2ngwIFKSkpSmzZt1L9/f02fPt2zzsyZM7Vq1SqtXLlSgwcP1oUXXqjWrVtrwoQJmjRpkp544glt3brV63FHjBihefPm6dtvv7X6JQGoJjp16qR27dppzJgxp1znscceU6tWrRQdHa3GjRvrnnvu0aFDhyRJa9as0W233SZJcjgccjgcGjRokCTvPQWffPKJXC6X9u7d6/XYqampio6O1uHDhyVJ+/bt06BBgxQfH686deqoc+fOWrduXVW/bNiAQgbbnXfeeVq7dq1+/fXXU67z6quvqkePHurQoUOFZcOHD1d0dLSWLFniNd6rVy917dq1wu4GAPCVw+HQs88+q9dff11btmypdJ2oqCi9/PLL+uabb7Rw4UKtWbNGDzzwgKTyQjdr1ixJUlZWlrKysjRjxowKj9GjRw+dd955FeaxRYsWqU+fPqpbt64KCwvVvXt3HTlyRB988IG2b9+unj176pprruE/ntUAhQy2mz59ugoLC9W4cWNdfPHFuuOOO7RkyRKVlpZ61vn+++/Vpk2bSu8fGRmppKQkff/99xWWPfvss3rvvff0f//3fwHLD6B6+/Of/6zevXvroYceqnT52LFj9ec//1lNmzZVjx49NGXKFC1dulSGYSgiIkKxsbGSpISEBCUkJHhun8jpdGrgwIF69dVXPWP79u3Txx9/rDvuuENS+dayw4cPKzU1VcnJyWrWrJkee+wxde7cWXPmzAnAK4eVKGSwXcuWLbVjxw5t3bpVw4YNU3FxsYYMGaKrrrpKhYWF5/TYl112mQYOHKiHH35YpmlWUWIANc1TTz2lDRs2aOXKlRWWvfPOO+rSpYsSExMVExOjAQMGqLi4WNnZ2X49xx133KGvv/5a27ZtkyQtWbJEDRs29OzW3Lx5s7Kzs1WvXj3FxMR4fq1fv167du069xcJW1HIEBQcDocuu+wy3X///Xr99df1ySefaOvWrXrjjTckSS1atNBXX31V6X2Lioq0Z88eXXzxxZUunzx5sr755psKuwIAwFctWrTQP//5Tz3yyCNeW+83bdqkfv36qUuXLlq2bJm2bduml156SZJUXFzs13O0atVKycnJWrx4sSRp8eLFGjhwoFwul6TyM9JbtWql9PR0r1/ffvut5s6dW0WvFHahkCEotWrVSpK0f/9+SdLAgQO1evVqbdq0qcK6M2bM0LFjxzRgwIBKH6tx48YaMWKEHnvsMRUVFQUuNIBqbcKECcrMzNTLL7/sGfv000/VoEEDTZo0SR06dFCLFi0qHJgfEREhSSorKzvjc9xxxx16/fXXtW3bNn3xxRe6/fbbPcuSk5OVkZGhunXrqlmzZl6/EhMTq+hVwi4UMtju73//u6ZNm6bPPvtMP/30kzZu3KjbbrtN4eHh6tWrl6TyA/e7deum66+/XgsWLNAPP/ygb7/9Vk888YTGjh2r8ePH6/LLLz/lc4wePVqFhYUVrlkGAL6Kj4/X6NGj9fzzz3vGLr74YuXk5Gj+/PnKyMjQ4sWL9eKLL3rd78ILL5QkrVy5Ujk5OTp69Ogpn+OWW27RgQMHdNddd6l9+/a65JJLPMsGDBigCy+8UL169dLHH3+sH3/8UZs2bdKUKVO0fPnyKn2tsB6FDLYwDENhYWGSpP/+7//Whx9+qL59+6pFixbq16+fIiIitHbtWrVu3VqSFB4ero8++kgjR47UtGnT1KpVK1155ZVKS0vT0qVL9cQTT5z2+erWrasJEyac8zFpAGq2kSNHqkGDBp7b//M//6PHHntMY8aMUdu2bbV06VI988wzXve54oorNHz4cP3zn/9Uw4YNNWzYsFM+vtvtVq9evZSenu61dUwqP4Fp7dq1Sk5O1p133qkWLVqob9+++s9//qMmTZpU7QuF5RwmRzrDBtdee63OP/98LViwwO4oAADYji1ksFRubq5WrFihtWvX6pprrrE7DgAAQSHM7gCoWfr166ddu3bpwQcfVP/+/e2OAwBAUGCXJQAAgM3YZQkAAGAzChkAAIDNKGQAAAA2C/mD+jMzM31eNzEx0a/1gwW5rUVua/mbuzpdkZz5K3iR21qhmlvyL/vp5i+2kAEAANiMQgYAAGAzChkAAIDNKGQAAAA2o5ABAADYjEIGAABgMwoZAACAzSy5DtmLL76obdu2KTY2VtOmTauw3DRNLViwQNu3b1etWrU0dOhQXXTRRVZEA2CTsn9cL0n65YQx19yV9oQ5AzvnsLLvdkgLZ2hv0TGZkdHSoOFytWxbJY8N4OyVzZosfbHpjzmsXQe5hj121o9nyRaybt26acyYMadcvn37dmVnZ+uFF17Q3XffrXnz5lkRC4BNjpcxX8ftZtccVvbdDmn6eClvv8yCo1Lefmn6+PJxALY5Xsa8fLGpfPwsWVLIWrdurZiYmFMu37Jli7p06SKHw6EWLVqooKBABw4csCIaAJyRbXPYwhmSUeY9ZpSVjwOwz8ll7EzjPgiKr07Kz89XgwYNPLfdbrfy8/NVv379CuumpaUpLS1NkjR16lS/v0YlVL92hdzWIndg/XKaZaHyGk7k6xzm7/y1t+iYzErGHUXHQup9CqWsJyK3tUIpdyDmsKAoZP5ISUlRSkqK5zbfBRe8yG2tUM19sjO9hlCatE/m7/xlRkZLBUcrHQ+VzzpUfy7Jba1QzV2Z072OoP8uy7i4OOXm5npu5+XlKS4uzsZEAOC7gM1hg4ZLTpf3mNNVPg7APu06+Dfug6AoZMnJyVq3bp1M09TOnTsVHR1d6e5KANXDqc6mDNazLM8kUHOYq2VbaeREyd1QjtoxkruhNHIiZ1kCNnMNe6xi+TrHsywt2WX5/PPP65tvvtGRI0d0zz336KabblJpaakk6dprr9Vll12mbdu26YEHHlBERISGDh1qRSwANjpevkJhV4Wdc5irZVtp6ryQeJ+AmuR4+aqqv5uWFLIRI0acdrnD4dCQIUOsiAIAfmMOAxBoQbHLEgAAoCajkAEAANiMQgYAAGAzChkAAIDNKGQAAAA2o5ABAADYjEIGAABgMwoZAACAzShkAAAANqOQAQAA2IxCBgAAYDMKGQAAgM0oZAAAADajkAEAANiMQgYAAGAzChkAAIDNKGQAAAA2o5ABAADYjEIGAABgszC7AwAIPkZOtrRiicyD+XLUi5N6D5AzPsHuWABQbVHIAHgxcrJlTh8v5WRLkkxJyvhexsiJlDIACBB2WQLwtmKJp4x5/L7FDAAQGBQyAF7Mg/l+jQMAzh2FDIAXR704v8YBAOeOQgbAW+8B0snHisUnlI8DAAKCg/oBeHHGJ8gYOZGzLAHAQhQyABU44xOkIf+yOwYA1BgUMgAAAD8dv17jvsKjMqJiznlPAoUMAADADyder7H4+OA5Xq+Rg/oBAAD8EYDrNVLIAAAA/BCI6zVSyAAAAPwQiOs1UsgAAAD8EYDrNXJQPwAAgB9OvF5jROFRFXOWJQAAgPWOX6+xYWKiMjMzz/3xqiATAAAAzgGFDAAAwGYUMgAAAJtRyAAAAGxGIQMAALCZZWdZpqena8GCBTIMQz169FCfPn28lufm5mr27NkqKCiQYRi69dZb1b59e6viAcApMX8BCDRLCplhGJo/f77Gjh0rt9utRx99VMnJybrgggs867z99tvq2LGjrr32Wu3du1dTpkxhQgN8UPbdDmnhDO0tOiYzMloaNFyulm3tjlVtMH8BsIIluyx3796thIQENWrUSGFhYerUqZM2b97stY7D4dCxY8ckSceOHVP9+vWtiAaEtLLvdkjTx0t5+2UWHJXy9kvTx5ePo0owfwGwgiVbyPLz8+V2uz233W63du3a5bVOv379NGnSJH344Yf67bffNG7cOCuiAaFt4QzJKPMeM8rKx6fOsydTNcP8BcAKQXOl/g0bNqhbt2667rrrtHPnTs2cOVPTpk2T0+m9ES8tLU1paWmSpKlTpyoxMdGv5/F3/WBBbmuFSu69RcdkVjLuKDoWMq9BCp33+1SYv06P3NYit/WqIrslhSwuLk55eXme23l5eYqL8/5G9NWrV2vMmDGSpBYtWqikpERHjhxRbGys13opKSlKSUnx3Pbn6woSq+jrDaxGbmuFUm4zMloqOFrpeKi8Bn/fb6snbeavc0Nua5Hbev5kP938ZckxZElJScrKytL+/ftVWlqqjRs3Kjk52WudBg0a6KuvvpIk7d27VyUlJapbt64V8YDQNWi45HR5jzld5eOoEsxfAKxgyRYyl8ulwYMHa/LkyTIMQ927d1fjxo2VmpqqpKQkJScn6/bbb9ecOXP03nvvSZKGDh0qh8NhRTwgZLlatlXZyInSwhlycJZlQDB/AbCCwzTNyg5BCRls8g9e5LZWTckdyseZnIz5K3iR21qhmlsKsV2WAAAAODUKGQAAgM0oZAAAADajkAEAANgsaC4MC+APZUtekta8/8dAt55yDbjHvkBAgBk52dKKJTIP5stRL07qPUDO+AS7Y+Es8FmeHQoZEGQqlDFJWvO+yiRKGaolIydb5vTxUk62JJV/+0TG9zJGTuQf8hDDZ3n22GUJBJuTy9iZxoFQt2KJ5x9wj9+3siDE8FmeNQoZAMBW5sF8v8YRvPgszx6FDABgK0e9OL/GEbz4LM8ehQwINt16+jcOhLreA6STjy+KTygfR2jhszxrHNQPBBnXgHtUJnGWJWoMZ3yCjJETOTOvGuCzPHsUMiAIuQbcI1HAUIM44xOkIf+yOwaqAJ/l2WGXJQAAgM0oZAAAADajkAEAANiMQgYAAGAzChkAAIDNKGQAAAA2o5ABAADYjEIGAABgMwoZAACAzShkAAAANqOQAQAA2IxCBgAAYDMKGQAAgM3C7A4AVAdl3+2QFs6QjhVI0bWlQcPlatnW7lgA4JOyTWulxbOkkhIpPFy6fZhcHbraHatGYQsZcI7KvtshTR8v5e2XCgvKf58+vnwcAIJc2aa10rxpUvFvkmmU/z5vWvk4LEMhA87VwhmSUeY9ZpSVjwNAsFs8y79xBASFDDhXxwr8GweAYFJS4t84AoJCBpyr6Nr+jQNAMAkP928cAXHGQpabm6v//Oc/yszMrLDs008/DUgoIKQMGi45Xd5jTlf5OAAEu9uH+TeOgDjtWZbp6emaPn26GjZsqKysLHXr1k2DBw+W01ne4+bOnaurr77akqBAsHK1bKuykRM5yxJASHJ16KoyibMsbXbaQvb6669r+PDhat++vQ4ePKiZM2fq6aef1kMPPaSwsDCZpmlVTiCouVq2labOszsGAJwVV4euEgXMVqfdZZmdna327dtLkurVq6cxY8YoMjJSU6ZM0W+//WZJQAAAgOrutIUsJiZGubm5ntsul0vDhw+X2+3Wk08+KcMwAh4QAACgujttIWvbtq3WrFnjNeZwODR06FD96U9/UgmnxAIAAJyz0x5DNmTIEJWVlVW67O6771bfvn0DEgoAAKAmOW0hCwsLU1jYqVdp0KBBlQcCAACoabgwLAAAgM0oZAAAADY77S7LqpSenq4FCxbIMAz16NFDffr0qbDOxo0b9eabb8rhcKhJkyYaPpwrnQOwH/MXgEDzqZCNGjVKTz/9dIXx0aNHa+rUqWe8v2EYmj9/vsaOHSu3261HH31UycnJuuCCCzzrZGVlafny5XryyScVExOjQ4cO+fEyACAwmL8QCoycbGnFEu0rPCojKkbqPUDO+AS7Y8EPPhWy7OzsCmOmaWrfvn0+Pcnu3buVkJCgRo0aSZI6deqkzZs3e01oq1at0l//+lfFxMRIkmJjY316bAAIJOYvBDsjJ1vm9PFSTraKjw9mfC9j5ERKWQg5bSGbNWuWJKm0tNTz5+NycnLUuHFjn54kPz9fbrfbc9vtdmvXrl1e6xz/8vJx48bJMAz169dPl156qU+PDwCBwvyFoLdiiZRz0oaT37eYaci/7MkEv522kB3/H+HJf3Y4HLr44ovVsWPHKgtiGIaysrI0YcIE5efna8KECXr22WdVu3Ztr/XS0tKUlpYmSZo6daoSExP9eh5/1w8W5LYWua0VqrmPY/46PXIH1r7Co39sGTtBROFRNQyR1yCFzvtdmarIftpC1q9fP0lS8+bNz+l/e3FxccrLy/PczsvLU1xcXIV1mjdvrrCwMDVs2FDnnXeesrKy1KxZM6/1UlJSlJKS4rl9/H+mvkhMTPRr/WBBbmuR21r+5rZ60mb+OjfkDjwjKqbS8eKomJB5DaH0fp/Mn+ynm798uuzFpZdeqszMTG3cuFGrV6/2+uWLpKQkZWVlaf/+/SotLdXGjRuVnJzstc6VV16pr7/+WpJ0+PBhZWVleW2VAwA7MH8h6PUeIJ18rFh8Qvk4QoZPB/W/8847evvtt9WkSRPVqlXLa9lf/vKXM97f5XJp8ODBmjx5sgzDUPfu3dW4cWOlpqYqKSlJycnJateunb744guNHDlSTqdTAwcOVJ06dc7uVQFAFWH+QrBzxifIGDlRWrFEEYVHVcxZliHJYZqmeaaVhgwZonHjxqlJkyZWZPILm/yDF7mtVVNyh/JxJidj/gpe5LZWqOaWLN5lGRERofPPP9+3ZAAAAPCLT4Wsf//+euWVV3TgwAEZhuH1CwAAAOfGp2PIXnzxRUnlFz88WWpqatUmAgAAqGF8KmQnXxQWAAAAVcenQhYfHy+p/OKHhw4dUv369QMaCgAAoCbxqZAVFBRo3rx5+vzzzxUWFqZXX31VW7Zs0e7du3XzzTcHOiMAAEC15tNB/XPnzlV0dLRefPFFhYWVd7gWLVpo48aNAQ0HAABQE/i0hWzHjh2aM2eOp4xJUt26dXXo0KGABQMAAKgpfNpCFh0drSNHjniN5ebmciwZAABAFfCpkPXo0UPTpk3TV199JdM0tXPnTs2ePVvXXHNNoPMBAABUez7tsuzdu7ciIiI0f/58lZWV6d///rdSUlLUs2fPQOcDAACo9nwqZA6HQz179qSAAQAABIBPhUyScnJy9NNPP6moqMhr/Oqrr67yUAAAADWJT4Vs2bJlevvtt3XBBRcoIiLCM+5wOChkAAAA58inQvbuu+9q6tSpuuCCCwKdBwAAoMbx6SzLmJgYz9cnAQAAoGr5tIVs0KBBmjNnjnr16qXY2FivZQ0aNAhIMAAAgJrCp0JWWlqqL7/8Uhs2bKiwLDU1tcpDAadS9slK6a1XJMOQnE7pxsFyXXO93bEA4IyMnGxpxRKZB/PlqBcn9R4gZ3yC3bFwlo5/nvsKj8qIijnnz9OnQjZv3jzdcsst6ty5s9dB/YCVyj5ZKb0x748Bw5DemKcyiVIGIKgZOdkyp4+XcrIlSaYkZXwvY+RESlkIOvHzLD4+eI6fp0/HkBmGoe7duysyMlJOp9PrF2CZt17xbxwAgsWKJZ4y5vH7FhaEoAB8nj41quuuu07Lly+XaZpn/UTAOTMM/8YBIEiYB/P9GkdwC8Tn6dMuyw8++EAHDx7UsmXLFBMT47Xs3//+91k/OeAXp7Py8sWWWgBBzlEvTpVt0nDUi7M8C85dID5PnwrZ/ffff9ZPAFSZGwd7H0N24jgABLPeA6SM7713c8UnlI8j9ATg8/SpkLVu3fqsnwCoKq5rrleZxFmWAEKOMz5BxsiJnGVZTZz4eUYUHlVxIM+yfOedd9S3b19Jp7+0Rf/+/c/6yQF/ua65XqKAAQhBzvgEaci/7I6BKnL882yYmKjMzMxzfrxTFrK8vLxK/wwAAICqdcpC9o9//ENS+SUvunTpoosvvljh4eGWBQMAAKgpznh6mtPp1NNPP00ZAwAACBCfrhfQqlUr7dy5M9BZAAAAaiSfzrKMj4/XlClTlJycLLfbLYfD4VnGQf0AAADnxqdCVlxcrCuuuEKSlJ/PVYUBAACqkk+FbOjQoYHOAQAAUGP5VMgk6ddff9Vnn32mQ4cO6a677lJmZqZKSkrUpEmTQOYDAACo9nw6qP+zzz7T+PHjlZ+fr3Xr1kmSCgsLtXjx4oCGAwAAqAl82kL2xhtvaNy4cWratKk+++wzSVKTJk30448/BjIbAABAjeDTFrJDhw5V2DXpcDi8zrYEAADA2fGpkF100UWeXZXHbdiwQc2aNQtIKAAAgJrEp12Wd955pyZNmqTVq1frt99+0+TJk5WZmamxY8cGOh8AAEC151MhO//88/X8889r69atuvzyy+V2u3X55ZcrMjIy0PkAAACqPZ8ve1GrVi21bNlS+fn5iouLo4wBAABUEZ8KWW5url544QXt2rVLtWvXVkFBgZo3b677779f8fHxgc4IAABQrfl0UP/s2bN10UUXacGCBZo3b54WLFigiy66SLNnz/b5idLT0zV8+HDdf//9Wr58+SnX+/zzz3XTTTdpz549Pj82AAQS8xeAQPOpkGVkZGjgwIGe3ZSRkZEaOHCgMjIyfHoSwzA0f/58jRkzRtOnT9eGDRu0d+/eCusVFhbqgw8+UPPmzf14CQg2Rk62jHnTtO/Rf8qYN01GTrbdkYCzxvwFwAo+FbLmzZtr9+7dXmN79uxRixYtfHqS3bt3KyEhQY0aNVJYWJg6deqkzZs3V1gvNTVVvXv3Vnh4uE+Pi+Bj5GTLnD5e5qa1Kv5yq8xNa2VOH08pQ8hi/gJgBZ+OIWvUqJGmTJmi9u3by+12Ky8vT9u3b9fVV1+t1NRUz3r9+/ev9P75+flyu92e2263W7t27fJaJyMjQ7m5uWrfvr1Wrlx5Nq8FwWDFEunk8pWTXT4+5F/2ZALOAfMXACv4VMhKSkrUoUMHSdLhw4cVHh6uK6+8UsXFxcrLyzvnEIZhaPHixRo6dOgZ101LS1NaWpokaerUqUpMTPTrufxdP1iESu59hUdVXMl4ROFRNQyR1yCFzvt9MnJbj/nrzMhtLXJbryqy+1TIfJloTicuLs6ruOXl5SkuLs5zu6ioSL/88oueeOIJSdLBgwf19NNPa9SoUUpKSvJ6rJSUFKWkpHhuZ2Zm+pwjMTHRr/WDRSjlNqJiKh0vjooJmdcQSu/3iWpKbqsnbeavc0Nua5Hbev5kP9385dMxZCd/bZIkmaapZcuW+RQgKSlJWVlZ2r9/v0pLS7Vx40YlJyd7lkdHR2v+/PmaPXu2Zs+erebNm1c6mSEE9B4gxSd4j8UnlI8DIYj5C4AVfNpC9tZbb2nr1q36xz/+oZiYGO3bt0+zZs2Sw+HQDTfccMb7u1wuDR48WJMnT5ZhGOrevbsaN26s1NRUJSUleU1uCG3O+AQZIydKK5YoovCoiqNipN4D5Dy5pAEhgvkLgBUcpmmaZ1qpqKhICxcu1BdffKGuXbvq448/1nXXXafevXvL6fRpI1vAsMk/eJHbWjUldygfZ3Iy5q/gRW5rhWpuyeJdlpGRkbr11ltVu3ZtLVu2TMnJyerTp4/tZQwAAKA68KlRbdu2TQ8//LDatGmjZ555RpmZmRo/frz2798f6HwAAADVnk+FbO7cubrvvvt055136k9/+pMmTpyodu3aafTo0YHOBwAAUO35dFD/M888o5iYPy5n4HQ6deONN6p9+/YBCwYAAFBT+FTIYmJi9OWXX+rTTz/V4cOHNXr0aO3Zs0eFhYWBzgcAAFDt+bTL8oMPPtDcuXOVmJiob7/9VpIUERGhpUuXBjQcAABATeBTIXv//fc1btw4rzMrzz///JA9RRUAACCY+FTICgsL1aBBA6+x0tJShYX5tMcTAAAAp+FTo2rVqpWWL1+uvn37esY++OADtWnTJmDBYJ+yJS9Ja97/Y6BbT7kG3GNfIADwQ9mmtdLiWVJJiRQeLt0+TK4OXe2OBZyWT4Vs8ODBeuqpp7Rq1SoVFRVp+PDhioqK4rIX1VCFMiZJa95XmUQpAxD0yjatleZN+2Og+Ddp3rTyOYxShiDmUyGrX7++pkyZoj179ignJ0dut1vNmjXjSv3V0cll7MRxChmAYLd41qnHKWQIYj4fBOZwONSsWTM1a9YskHkAADh7JSX+jQNBgk1cAIDqIzzcv3EgSFDI4K1bT//GASCY3D7Mv3EgSHDdCnhxDbhHZRJnWQIISa4OXcvnMM6yRIihkKEC14B7OIAfQMhydejKAfwIOeyyBAAAsBmFDAAAwGYUMgAAAJtRyAAAAGxGIQMAALAZhQwAAMBmFDIAAACbUcgAAABsRiEDAACwGYUMAADAZhQyAAAAm1HIAAAAbEYhAwAAsFmY3QFQNcq+2yEtnCEdK5Cia0uDhsvVsq3dsQDAJ8xhqOnYQlYNlH23Q5o+XsrbLxUWlP8+fXz5OAAEOeYwgEJWPSycIRll3mNGWfk4AAQ75jCAQlYtHCvwbxwAgglzGEAhqxaia/s3DgDBhDkMoJBVC4OGS06X95jTVT4OAMGOOQygkFUHrpZtpZETJXdDKap2+e8jJ3KGEoCQwBwGcNmLasPVsq00dZ7dMQDgrDjc8VKzVjIP5stRL05yx9sdCbAUhQwAYCsjJ1vm9PFSTrYkyZSkjO9ljJwoZ3yCrdkAq7DLEgBgrxVLPGXMIye7fByoIShkAABbmQfz/RoHqiPLdlmmp6drwYIFMgxDPXr0UJ8+fbyWv/vuu1q1apVcLpfq1q2re++9V/HxHEMAwH7MX4HlqBdXvpuyknGgprCkkBmGofnz52vs2LFyu9169NFHlZycrAsuuMCzTtOmTTV16lTVqlVLH3/8sV577TWNHDnSinhBy/h9k/2+wqMyomKk3gM4ngKwGPPX2fFr/uo9QMr43nu3ZXxC+ThQQ1hSyHbv3q2EhAQ1atRIktSpUydt3rzZa0K75JJLPH9u3ry51q9fb0W0oHXiQa7Fxwc5yBWwHPOX//ydv5zxCTJGTpRWLPnjLEv+A4oaxpJjyPLz8+V2uz233W638vNPfWzA6tWrdemll1qQLIhxkCsQFJi/zsJZzF/O+AQ5h/xLrocmyznkX5Qx1DhBd9mLdevWKSMjQ48//nily9PS0pSWliZJmjp1qhITE/16fH/Xt8u+wqN//M/yBBGFR9UwRF6DFDrv98nIba1QzX0y5q9yzF/2Irf1qiK7JYUsLi5OeXl5ntt5eXmKi6t4sOaXX36pZcuW6fHHH1d4eHilj5WSkqKUlBTP7czMTJ9zJCYm+rW+nYyomErHi6NiQuY1hNL7fSJyW8vf3FZP2sxf/mP+sg+5redP9tPNX5bsskxKSlJWVpb279+v0tJSbdy4UcnJyV7r/PDDD5o7d65GjRql2NhYK2IFt94Dyg9qPREHuQKWY/46C8xfgN8s2ULmcrk0ePBgTZ48WYZhqHv37mrcuLFSU1OVlJSk5ORkvfbaayoqKtJzzz0nSWrQoIEeeeQRK+IFpRMPco0oPKpizrIEbMH85T/mL8B/DtM0K7v8S8iorpv8T0Rua5HbWsG+yzKQmL+CF7mtFaq5parbZRl0B/XXJGXf7ZAWzpCOFUjRtaVBw8u/JBwAgtzx64xxmQqgalDIbFL23Q5p+njJKCsfKCyQpo9X2ciJlDIAQY0vAweqHt9laZeFM/4oY8cZZeXjABDMuE4iUOUoZHY5VuDfOAAECb4MHKh6FDK7RNf2bxwAgsSpvvSbLwMHzh6FzC6DhktOl/eY01U+DgDBjOuMAVWOg/pt4mrZVmUjJ3KWJYCQw5eBA1WPQmYjV8u20tR5dscAAL854xOkIf+yOwZQbbDLEgAAwGYUMgAAAJtRyAAAAGxGIQMAALAZhQwAAMBmFDIAAACbUcgAAABsRiEDAACwGYUMAADAZhQyAAAAm1HIAAAAbEYhAwAAsBmFDAAAwGYUMgAAAJtRyAAAAGxGIQMAALAZhQwAAMBmFDIAAACbUcgAAABsRiEDAACwGYUMAADAZhQyAAAAm1HIAAAAbEYhAwAAsBmFDAAAwGYUMgAAAJtRyAAAAGxGIQMAALAZhQwAAMBmFDIAAACbUcgAAABsRiEDAACwGYUMAADAZmFWPVF6eroWLFggwzDUo0cP9enTx2t5SUmJZs2apYyMDNWpU0cjRoxQw4YNq+S5y/5xvSTplxPGXHNXVsljA6j+mL9Ci5GTLa1YIvNgvhz14qTeA+SMT7A7FnBalmwhMwxD8+fP15gxYzR9+nRt2LBBe/fu9Vpn9erVql27tmbOnKlevXppyZIlVfLcxyczX8cB4ETMX6HFyMmWOX28zE1rpe93yNy0Vub08eUlDQhilhSy3bt3KyEhQY0aNVJYWJg6deqkzZs3e62zZcsWdevWTZJ01VVX6auvvpJpmlbEA4BTYv4KMSuWSCeXr9+3mAHBzJJdlvn5+XK73Z7bbrdbu3btOuU6LpdL0dHROnLkiOrWreu1XlpamtLS0iRJU6dOVWJi4mmf+5fTLDvTfYNJKGU9EbmtRe6qx/x17qzMuq/wqIorGY8oPKqGfuYIpff4ROS2hmmaysnJ0a+//lphWXh4uOLj4+VwOHx+PMuOIasqKSkpSklJ8dzOzMw868c6l/taKTExMWSynojc1qopuUNt0j4R81fgGVExlY4XR8X4/XMWKu/xichtncLCQoWHhysqKkolJSVey4qKipSRkaGoqCiv8dPNX5bssoyLi1NeXp7ndl5enuLi4k65TllZmY4dO6Y6depYEQ8ATon5K8T0HiCdfAB/fEL5OFCFDMNQWFjl27XCwsJkGIZfj2dJIUtKSlJWVpb279+v0tJSbdy4UcnJyV7rXH755VqzZo0k6fPPP1ebNm382tR3Kqc6G4mzlAD4gvkrtDjjE+QYOVGODl2li9vK0aGrHCMncpYlqtyZ/o77OwdYssvS5XJp8ODBmjx5sgzDUPfu3dW4cWOlpqYqKSlJycnJ+stf/qJZs2bp/vvvV0xMjEaMGFF1z//75BWKm0QB2Iv5K/Q44xOkIf+yOwbgF8uOIWvfvr3at2/vNda/f3/PnyMiIvTggw9aFQcAfMb8BSDQuFI/AACAn850aRt/L31DIQMAAPCT0+lUaWlppctKS0vldPpXsULushcAAAB2i4yMVFFRkVwulwoLCz3jpmnK6XQqMjLSr8ejkAEAAPjJ4XAoKiqqyk64YZclAACAzShkAAAANqOQAQAA2Mxh+nteJgAAAKpUjdpCNnr0aLsjnBVyW4vc1grV3FYL1feJ3NYit/WqKnuNKmQAAADBiEIGAABgsxpVyFJSUuyOcFbIbS1yWytUc1stVN8ncluL3Narquwc1A8AAGCzGrWFDAAAIBhVu69OSk9P14IFC2QYhnr06KE+ffp4LS8pKdGsWbOUkZGhOnXqaMSIEWrYsKE9YU9wptzvvvuuVq1aJZfLpbp16+ree+9VfHy8PWFPcqbsx33++ed67rnnNGXKFCUlJVkbshK+5N64caPefPNNORwONWnSRMOHD7c+6EnOlDs3N1ezZ89WQUGBDMPQrbfeqvbt29sT9ncvvviitm3bptjYWE2bNq3CctM0tWDBAm3fvl21atXS0KFDddFFF9mQ1H7MYdZi/rJWKM5fkkVzmFmNlJWVmcOGDTOzs7PNkpIS86GHHjJ/+eUXr3U+/PBDc86cOaZpmuann35qPvfcc3ZE9eJL7h07dphFRUWmaZrmRx99FBS5TdO37KZpmseOHTPHjx9vjhkzxty9e7cNSb35kjszM9N8+OGHzSNHjpimaZoHDx60I6oXX3K/9NJL5kcffWSapmn+8ssv5tChQ+2I6uXrr7829+zZYz744IOVLt+6das5efJk0zAM8/vvvzcfffRRixMGB+YwazF/WStU5y/TtGYOq1a7LHfv3q2EhAQ1atRIYWFh6tSpkzZv3uy1zpYtW9StWzdJ0lVXXaWvvvpKps2H0fmS+5JLLlGtWrUkSc2bN1d+fr4dUSvwJbskpaamqnfv3goPD7chZUW+5F61apX++te/KiYmRpIUGxtrR1QvvuR2OBw6duyYJOnYsWOqX7++HVG9tG7d2vM+VmbLli3q0qWLHA6HWrRooYKCAh04cMDChMGBOcxazF/WCtX5S7JmDqtWhSw/P19ut9tz2+12V/hLf+I6LpdL0dHROnLkiKU5T+ZL7hOtXr1al156qQXJzsyX7BkZGcrNzQ2Kzc7H+ZI7MzNTWVlZGjdunB577DGlp6dbnLIiX3L369dP69ev1z333KMpU6Zo8ODBVsf0W35+vho0aOC5faa/A9UVc5i1mL+sVV3nL6lq5rBqVchqgnXr1ikjI0PXX3+93VF8YhiGFi9erNtvv93uKH4zDENZWVmaMGGChg8frjlz5qigoMDuWGe0YcMGdevWTS+99JIeffRRzZw5U4Zh2B0LkBRacxjzl/Vq8vxVrQpZXFyc8vLyPLfz8vIUFxd3ynXKysp07Ngx1alTx9KcJ/MltyR9+eWXWrZsmUaNGhU0m87PlL2oqEi//PKLnnjiCd13333atWuXnn76ae3Zs8eOuB6+/qwkJycrLCxMDRs21HnnnaesrCyro1bIdKbcq1evVseOHSVJLVq0UElJie1bUM4kLi5Oubm5ntun+jtQ3TGHWYv5y1rVdf6SqmYOq1aFLCkpSVlZWdq/f79KS0u1ceNGJScne61z+eWXa82aNZLKz5pp06aNHA6HDWn/4EvuH374QXPnztWoUaOC4liA486UPTo6WvPnz9fs2bM1e/ZsNW/eXKNGjbL9LCVf3vMrr7xSX3/9tSTp8OHDysrKUqNGjeyI6+FL7gYNGuirr76SJO3du1clJSWqW7euHXF9lpycrHXr1sk0Te3cuVPR0dFBc+yIlZjDrMX8Za3qOn9JVTOHVbsLw27btk2LFi2SYRjq3r27+vbtq9TUVCUlJSk5OVnFxcWaNWuWfvjhB8XExGjEiBG2/5D6kvvJJ5/Uzz//rHr16kkq/6F95JFH7A39uzNlP9Hjjz+u2267zfYJTTpzbtM0tXjxYqWnp8vpdKpv377q3Lmz3bHPmHvv3r2aM2eOioqKJEkDBw5Uu3btbM38/PPP65tvvtGRI0cUGxurm266SaWlpZKka6+9VqZpav78+friiy8UERGhoUOHBsXPiB2Yw6zF/BVcuYNx/pKsmcOqXSEDAAAINdVqlyUAAEAoopABAADYjEIGAABgMwoZAACAzShkAAAANqOQIei9/PLLeuutt+yOAQBAwHDZC+AU3njjDWVnZ+uBBx445Toffvih1qxZo59//lmdO3fWfffdZ2FCAEB1EWZ3AOB0DMOQ0xm8G3Lr16+vvn376osvvlBxcbHdcQAAIYpChlPKz8/XK6+8om+//VaRkZHq1auXevbsqaNHj+qhhx7SkCFDlJycrKKiIj388MO68cYb1bVrV82ePVvh4eHat2+fdu3apQsvvFDDhg1TfHy8JOnXX3/VK6+8ooyMDNWtW1f9+/dXp06dJEmzZ89WRESEcnNz9c033+jhhx/W+vXr5Xa7dfPNN+vrr7/WzJkz9be//U3/+7//K6fTqSFDhigsLEyLFi3S4cOHdd1116lv376SygvdypUrtWrVKhUUFOiSSy7R3XffrZiYGO3fv1/Dhg3T0KFDlZqaquLiYvXq1Ut9+/ZVenq6li1bJknavHmzEhIS9Mwzz1R4jzp06CBJysjI8PqONgAA/BG8mx5gK8Mw9NRTT6lp06aaM2eOxo8fr/fff1/p6emKiYnRvffeqzlz5ujQoUNauHChmjZtqq5du3ru/+mnn+rvf/+75s+fr6ZNm+qFF16QVP5lvZMmTdLVV1+tefPmacSIEZo/f7727t3rdd8bbrhBixYtUsuWLStkO3jwoEpKSvTSSy/ppptu0pw5c7R+/XpNnTpVEydO1Ntvv639+/dLKt+luHnzZj3++OOaM2eOYmJiNG/ePK/H++677zRjxgyNGzdOb731lvbu3atLL71UN9xwgzp27KhXX3210jIGAEBVoZChUnv27NHhw4d14403KiwsTI0aNVKPHj20ceNGSVK7du3UsWNHTZw4Udu3b9fdd9/tdf/27durdevWCg8P1y233KKdO3cqNzdX27ZtU3x8vLp37y6Xy6ULL7xQHTp00Geffea57xVXXKGWLVvK6XQqIiKiQjaXy6W+ffsqLCxMnTt31pEjR9SzZ09FRUWpcePGuuCCC/Tjjz9Kkj755BPdfPPNcrvdCg8PV79+/bRp0yaVlZV5Hq9fv36KiIhQ06ZN1aRJE/30008BeEcBADg1dlmiUjk5OTpw4IAGDRrkGTMMQ61atfLcTklJ0YcffqgbbrhBderU8bq/2+32/DkyMlIxMTE6cOCAcnJytGvXLq/HLSsrU5cuXSq9b2Xq1KnjOa7seGGLjY31LI+IiPB8MW1OTo6effZZORwOz3Kn06lDhw55bh//smNJqlWrlue+AABYhUKGSjVo0EANGzb07Go8mWEYmjNnjrp06aKPP/5Y3bt3V0JCgmf5icdTFRUV6ejRo6pfv77cbrdat26tcePGnfK5TyxP58rtduvee++tdNfn8d2aVuQAAOB02GWJSjVr1kxRUVFavny5iouLZRiGfv75Z+3evVuS9M4778jhcGjo0KG67rrrNGvWLBmG4bn/9u3b9d1336m0tFRLly5VixYt1KBBA11++eXKysrSunXrVFpaqtLSUu3evdvrGLKqdM0112jp0qXKycmRJB0+fFibN2/26b6xsbHKycnxel0nKysr87w/hmGouLjYa3coAAC+YAsZKuV0OvXII49o8eLFuu+++1RaWqrExET1799fGRkZeu+99zRlyhQ5nU716dNH27dv1/Llyz1nN3bu3Flvvvmmdu7cqYsuukj333+/JCkqKkpjx47VokWLtGjRIpmmqSZNmuiOO+4IyOvo2bOnJGnSpEk6cOCAYmNj1bFjR11xxRVnvG/Hjh21fv163XXXXWrYsKGeeuqpCuu8/fbbXhetXb9+vW688UbddNNNVfciAADVHheGRZWbPXu25zIVAADgzNhlCQAAYDMKGQAAgM3YZQkAAGAztpABAADYjEIGAABgMwoZAACAzShkAAAANqOQAQAA2IxCBgAAYLP/B5nB+ofFwG+kAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.style.use('ggplot') \n", + "fig,ax = plt.subplots(1,2,figsize=(10, 5))\n", + "ax[0].scatter(read_score(\"prompt_to_json_eli5\"),read_score(\"prompt_to_json_eli5_1\"),marker=\"o\")\n", + "ax[0].set_title(\"JSON\")\n", + "ax[1].scatter(read_score(\"native_eli5\"),read_score(\"native_eli5_1\"),marker=\"o\")\n", + "ax[1].set_title(\"Native\")\n", + "ax[0].set_xlabel(\"experiment 1\")\n", + "ax[0].set_ylabel(\"experiment 2\")\n", + "\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "b7e6a507", + "metadata": {}, + "source": [ + "## Self consistency \n", + "The idea is to select the sample with highest similarity to other samples, among k generated samples\n", + "1) n-gram + cosine similarity\n", + "\n", + "2) embeddings + cosine similarity\n", + "\n", + "### Use case\n", + "- building this to select the best from k generations that are all json\n", + "- For each json in output\n", + " - for each key in json\n", + " - select sample with that maximises sim score " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77aff1ff", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ragas", + "language": "python", + "name": "ragas" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/ragas/metrics/_context_recall.py b/src/ragas/metrics/_context_recall.py index d0165fe2a..417cd6ac4 100644 --- a/src/ragas/metrics/_context_recall.py +++ b/src/ragas/metrics/_context_recall.py @@ -1,8 +1,10 @@ from __future__ import annotations +import re import typing as t from dataclasses import dataclass +import numpy as np from datasets import Dataset from langchain.callbacks.manager import CallbackManager, trace_as_chain_group from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate @@ -11,19 +13,51 @@ CONTEXT_RECALL_RA = HumanMessagePromptTemplate.from_template( """ -Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. -Think in steps and reason before coming to conclusion. +Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Output json with reason. + +question: What can you tell me about albert Albert Einstein? context: Albert Einstein (14 March 1879 – 18 April 1955) was a German-born theoretical physicist,widely held to be one of the greatest and most influential scientists of all time. Best known for developing the theory of relativity, he also made important contributions to quantum mechanics, and was thus a central figure in the revolutionary reshaping of the scientific understanding of nature that modern physics accomplished in the first decades of the twentieth century. His mass–energy equivalence formula E = mc2, which arises from relativity theory, has been called "the world's most famous equation". He received the 1921 Nobel Prize in Physics "for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect", a pivotal step in the development of quantum theory. His work is also known for its influence on the philosophy of science. In a 1999 poll of 130 leading physicists worldwide by the British journal Physics World, Einstein was ranked the greatest physicist of all time. His intellectual achievements and originality have made Einstein synonymous with genius. answer: Albert Einstein born in 14 March 1879 was German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. He received the 1921 Nobel Prize in Physics "for his services to theoretical physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895 -classification -1. Albert Einstein born in 14 March 1879 was German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time. The date of birth of Einstein is mentioned clearly in the context. So [Attributed] -2. He received the 1921 Nobel Prize in Physics "for his services to theoretical physics. The exact sentence is present in the given context. So [Attributed] -3. He published 4 papers in 1905. There is no mention about papers he wrote in given the context. So [Not Attributed] -4. Einstein moved to Switzerland in 1895. There is not supporting evidence for this in the given the context. So [Not Attributed] - +classification: +[ + {{ "statement_1":"Albert Einstein, born on 14 March 1879, was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time.", + "reason": "The date of birth of Einstein is mentioned clearly in the context.", + "Attributed": "Yes" + }}, + {{ + "statement_2":"He received the 1921 Nobel Prize in Physics 'for his services to theoretical physics.", + "reason": "The exact sentence is present in the given context.", + "Attributed": "Yes" + }}, + {{ + "statement_3": "He published 4 papers in 1905.", + "reason": "There is no mention about papers he wrote in the given context.", + "Attributed": "No" + }}, + {{ + "statement_4":"Einstein moved to Switzerland in 1895.", + "reason": "There is no supporting evidence for this in the given context.", + "Attributed": "No" + }} +] + +question: who won 2020 icc world cup? +context: Who won the 2022 ICC Men's T20 World Cup? +The 2022 ICC Men's T20 World Cup, held from October 16 to November 13, 2022, in Australia, was the eighth edition of the tournament. Originally scheduled for 2020, it was postponed due to the COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets in the final to clinch their second ICC Men's T20 World Cup title. +answer: England +classification: +[ + {{ + "statement_1":"England won the 2022 ICC Men's T20 World Cup.", + "reason": "From context it is clear that England defeated Pakistan to win the World Cup.", + "Attributed": "Yes" + }} +] + +question:{question} context:{context} -answer:{ground_truth} +answer:{answer} classification: """ # noqa: E501 ) @@ -44,7 +78,7 @@ class ContextRecall(MetricWithLLM): """ name: str = "context_recall" - evaluation_mode: EvaluationMode = EvaluationMode.gc + evaluation_mode: EvaluationMode = EvaluationMode.qcg batch_size: int = 15 def _score_batch( @@ -53,17 +87,22 @@ def _score_batch( callbacks: t.Optional[CallbackManager] = None, callback_group_name: str = "batch", ) -> list: - verdict_token = "[Attributed]" prompts = [] - ground_truths, contexts = dataset["ground_truths"], dataset["contexts"] + question, ground_truths, contexts = ( + dataset["question"], + dataset["ground_truths"], + dataset["contexts"], + ) with trace_as_chain_group( callback_group_name, callback_manager=callbacks ) as batch_group: - for gt, ctx in zip(ground_truths, contexts): + for qstn, gt, ctx in zip(question, ground_truths, contexts): gt = "\n".join(gt) if isinstance(gt, list) else gt ctx = "\n".join(ctx) if isinstance(ctx, list) else ctx - human_prompt = CONTEXT_RECALL_RA.format(context=ctx, ground_truth=gt) + human_prompt = CONTEXT_RECALL_RA.format( + question=qstn, context=ctx, answer=gt + ) prompts.append(ChatPromptTemplate.from_messages([human_prompt])) responses: list[list[str]] = [] @@ -75,12 +114,17 @@ def _score_batch( responses = [[i.text for i in r] for r in results.generations] scores = [] for response in responses: - sentences = response[0].split("\n") - denom = len(sentences) - numerator = sum( - bool(sentence.find(verdict_token) != -1) for sentence in sentences - ) - scores.append(numerator / denom) + pattern = "\[\s*\{.*?\}(\s*,\s*\{.*?\})*\s*\]" + match = re.search(pattern, response[0].replace("\n", "")) + if match: + response = eval(response[0]) + denom = len(response) + numerator = sum( + item.get("Attributed").lower() == "yes" for item in response + ) + scores.append(numerator / denom) + else: + scores.append(np.nan) return scores diff --git a/src/ragas/metrics/base.py b/src/ragas/metrics/base.py index 94c1612ee..8afb43cde 100644 --- a/src/ragas/metrics/base.py +++ b/src/ragas/metrics/base.py @@ -40,7 +40,7 @@ def make_batches(total_size: int, batch_size: int) -> list[range]: return batches -EvaluationMode = Enum("EvaluationMode", "qac qa qc gc ga qga") +EvaluationMode = Enum("EvaluationMode", "qac qa qc gc ga qga qcg") @dataclass diff --git a/src/ragas/validation.py b/src/ragas/validation.py index 18c4d853c..f5aa784c2 100644 --- a/src/ragas/validation.py +++ b/src/ragas/validation.py @@ -43,6 +43,7 @@ def validate_column_dtypes(ds: Dataset): EvaluationMode.gc: ["ground_truths", "contexts"], EvaluationMode.ga: ["ground_truths", "answer"], EvaluationMode.qga: ["question", "ground_truths", "answer"], + EvaluationMode.qcg: ["question", "contexts", "ground_truths"], }