Tutorials/Housing_data_Example.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install ProTaska-GPT"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AOsiqm8AMyxs",
        "outputId": "d5864909-24df-473c-c44e-cfeb153edf1f"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting ProTaska-GPT\n",
            "  Downloading ProTaska_GPT-0.0.12-py3-none-any.whl (10 kB)\n",
            "Collecting langchain (from ProTaska-GPT)\n",
            "  Downloading langchain-0.0.202-py3-none-any.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from ProTaska-GPT) (1.22.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from ProTaska-GPT) (1.5.3)\n",
            "Collecting colorama (from ProTaska-GPT)\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Collecting wikipedia (from ProTaska-GPT)\n",
            "  Downloading wikipedia-1.4.0.tar.gz (27 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting openai (from ProTaska-GPT)\n",
            "  Downloading openai-0.27.8-py3-none-any.whl (73 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.6/73.6 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets (from ProTaska-GPT)\n",
            "  Downloading datasets-2.13.0-py3-none-any.whl (485 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m485.6/485.6 kB\u001b[0m \u001b[31m39.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tiktoken (from ProTaska-GPT)\n",
            "  Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m53.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->ProTaska-GPT) (9.0.0)\n",
            "Collecting dill<0.3.7,>=0.3.0 (from datasets->ProTaska-GPT)\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets->ProTaska-GPT) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets->ProTaska-GPT) (4.65.0)\n",
            "Collecting xxhash (from datasets->ProTaska-GPT)\n",
            "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets->ProTaska-GPT)\n",
            "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets->ProTaska-GPT) (2023.4.0)\n",
            "Collecting aiohttp (from datasets->ProTaska-GPT)\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m73.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub<1.0.0,>=0.11.0 (from datasets->ProTaska-GPT)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets->ProTaska-GPT) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets->ProTaska-GPT) (6.0)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain->ProTaska-GPT) (2.0.10)\n",
            "Collecting async-timeout<5.0.0,>=4.0.0 (from langchain->ProTaska-GPT)\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting dataclasses-json<0.6.0,>=0.5.7 (from langchain->ProTaska-GPT)\n",
            "  Downloading dataclasses_json-0.5.8-py3-none-any.whl (26 kB)\n",
            "Collecting langchainplus-sdk>=0.0.9 (from langchain->ProTaska-GPT)\n",
            "  Downloading langchainplus_sdk-0.0.10-py3-none-any.whl (21 kB)\n",
            "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain->ProTaska-GPT) (2.8.4)\n",
            "Collecting openapi-schema-pydantic<2.0,>=1.2 (from langchain->ProTaska-GPT)\n",
            "  Downloading openapi_schema_pydantic-1.2.4-py3-none-any.whl (90 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain->ProTaska-GPT) (1.10.7)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain->ProTaska-GPT) (8.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->ProTaska-GPT) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->ProTaska-GPT) (2022.7.1)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->ProTaska-GPT) (2022.10.31)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from wikipedia->ProTaska-GPT) (4.11.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ProTaska-GPT) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ProTaska-GPT) (2.0.12)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets->ProTaska-GPT)\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0 (from aiohttp->datasets->ProTaska-GPT)\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m30.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp->datasets->ProTaska-GPT)\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp->datasets->ProTaska-GPT)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting marshmallow<4.0.0,>=3.3.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain->ProTaska-GPT)\n",
            "  Downloading marshmallow-3.19.0-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting marshmallow-enum<2.0.0,>=1.5.1 (from dataclasses-json<0.6.0,>=0.5.7->langchain->ProTaska-GPT)\n",
            "  Downloading marshmallow_enum-1.5.1-py2.py3-none-any.whl (4.2 kB)\n",
            "Collecting typing-inspect>=0.4.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain->ProTaska-GPT)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets->ProTaska-GPT) (3.12.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets->ProTaska-GPT) (4.5.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->ProTaska-GPT) (1.16.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets->ProTaska-GPT) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets->ProTaska-GPT) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets->ProTaska-GPT) (3.4)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain->ProTaska-GPT) (2.0.2)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->wikipedia->ProTaska-GPT) (2.4.1)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain->ProTaska-GPT)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Building wheels for collected packages: wikipedia\n",
            "  Building wheel for wikipedia (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wikipedia: filename=wikipedia-1.4.0-py3-none-any.whl size=11680 sha256=a93a76c93c0d0fbd04eca13912edbcdce4663bcfc92e7c01d49d7342978f3ead\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/b6/c5/93f3dec388ae76edc830cb42901bb0232504dfc0df02fc50de\n",
            "Successfully built wikipedia\n",
            "Installing collected packages: xxhash, mypy-extensions, multidict, marshmallow, frozenlist, dill, colorama, async-timeout, yarl, wikipedia, typing-inspect, tiktoken, openapi-schema-pydantic, multiprocess, marshmallow-enum, langchainplus-sdk, huggingface-hub, aiosignal, dataclasses-json, aiohttp, openai, langchain, datasets, ProTaska-GPT\n",
            "Successfully installed ProTaska-GPT-0.0.12 aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 colorama-0.4.6 dataclasses-json-0.5.8 datasets-2.13.0 dill-0.3.6 frozenlist-1.3.3 huggingface-hub-0.15.1 langchain-0.0.202 langchainplus-sdk-0.0.10 marshmallow-3.19.0 marshmallow-enum-1.5.1 multidict-6.0.4 multiprocess-0.70.14 mypy-extensions-1.0.0 openai-0.27.8 openapi-schema-pydantic-1.2.4 tiktoken-0.4.0 typing-inspect-0.9.0 wikipedia-1.4.0 xxhash-3.2.0 yarl-1.9.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from protaska.describer import describe_dataset\n",
        "from protaska.ideate import main as chatbot"
      ],
      "metadata": {
        "id": "xxv19cDMTAoF"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "openai_key = 'Enter **secret key**'\n",
        "importer_type = \"LocalDatasetImporter\"\n",
        "dataset_key = ''\n",
        "destination_path = './sample_data/'"
      ],
      "metadata": {
        "id": "7QiSxvLOTBMn"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nL2nwHyQi_KS",
        "outputId": "0a8f031c-c85b-446b-9c3e-626a51750f69"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "german_credit_data.csv\tsample_data\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "description, dataloader_obj = describe_dataset(openai_key, importer_type, destination_path, dataset_key)\n",
        "description"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "id": "WHJ7hs_yTC7n",
        "outputId": "cde109b5-6df2-4310-d22a-81a5cce6a099"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Description: The dataset is a CSV file containing data on housing prices in California, with features such as longitude, latitude, housing median age, total rooms, total bedrooms, population, households, median income, and median house value. It is suitable for regression tasks using machine learning algorithms such as linear regression or decision trees. The example data shows the longitude and latitude of housing locations and their corresponding median house values. The data is imported locally and contains numerical values for each feature.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VQhnWSiPMxPF",
        "outputId": "8654a3c0-b1bb-43b1-c26f-ac6389ede6d7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ProTaska:\t Hello! Based on the description of the dataset you provided, there are several ML/DS tasks that can be performed. Here are some examples:\n",
            "\n",
            "1. Regression Analysis: This task involves predicting a continuous numerical value, such as the median house value in this dataset. You can use machine learning algorithms such as linear regression or decision trees to build a model that can predict the median house value based on the other features in the dataset.\n",
            "\n",
            "2. Clustering: This task involves grouping similar data points together based on their features. In this dataset, you could use clustering algorithms such as K-means or DBSCAN to group similar housing locations together based on their longitude, latitude, and other features.\n",
            "\n",
            "3. Feature Engineering: This task involves creating new features from the existing ones to improve the performance of your machine learning model. For example, you could create a new feature that represents the ratio of total bedrooms to total rooms, which may be a better predictor of median house value than either feature alone.\n",
            "\n",
            "4. Data Visualization: This task involves creating visualizations of the data to better understand patterns and relationships between the features. You could use libraries such as Matplotlib or Seaborn to create scatter plots or heatmaps of the data.\n",
            "\n",
            "If you have any specific questions or tasks in mind, feel free to ask and I can provide more information on how to approach them.\n",
            "ProTaska-Source:\t Connected to Wikipedia for external information!\n",
            "\n",
            "Human (input 'break' or 'exit' to stop the loop):\tcreate a script for clustering\n",
            "ProTaska:\t Here is a sample script for clustering the housing data using KMeans algorithm:\n",
            "\n",
            "import pandas as pd\n",
            "from sklearn.cluster import KMeans\n",
            "from sklearn.preprocessing import StandardScaler\n",
            "\n",
            "# Load data\n",
            "housing_data = pd.read_csv('./sample_data/housing.csv')\n",
            "\n",
            "# Preprocess data\n",
            "numerical_data = housing_data.drop(['ocean_proximity'], axis=1)\n",
            "scaler = StandardScaler()\n",
            "scaled_data = scaler.fit_transform(numerical_data)\n",
            "\n",
            "# Cluster data\n",
            "kmeans = KMeans(n_clusters=3)\n",
            "kmeans.fit(scaled_data)\n",
            "\n",
            "# Get cluster labels\n",
            "labels = kmeans.labels_\n",
            "\n",
            "# Add labels to original data\n",
            "housing_data['cluster'] = labels\n",
            "\n",
            "# Save data\n",
            "housing_data.to_csv('./sample_data/housing_clustered.csv', index=False)\n",
            "\n",
            "This script loads the housing data, preprocesses it by scaling the numerical features, clusters the data using KMeans algorithm with 3 clusters, adds the cluster labels to the original data, and saves the data with cluster labels to a new CSV file.\n",
            "\n",
            "Human (input 'break' or 'exit' to stop the loop):\tbreak\n",
            "ProTaska:\tStopping Execution!\n"
          ]
        }
      ],
      "source": [
        "chatbot(openai_key, description, dataloader_obj.superficial_meta_data, agent_verbose=False)"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "_zl-0STPNJc8"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}