Sentimental_Analysis_with_BERT on Vertex AI.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Movie Reviews"
      ],
      "metadata": {
        "id": "1F7ld_Gy6upG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sdkrj4mqsYmt"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import warnings\n",
        "\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_XgTpm9ZxoN9",
        "tags": []
      },
      "outputs": [],
      "source": [
        "import datetime\n",
        "import shutil\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text\n",
        "from google.cloud import aiplatform\n",
        "from official.nlp import optimization  # to create AdamW optmizer\n",
        "\n",
        "tf.get_logger().setLevel(\"ERROR\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pOdqCMoQDRJL",
        "tags": []
      },
      "outputs": [],
      "source": [
        "url = \"https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\"\n",
        "\n",
        "# Set a path to a folder outside the git repo. This is important so data won't get indexed by git on Jupyter lab\n",
        "path = \"/home/jupyter/\"\n",
        "\n",
        "dataset = tf.keras.utils.get_file(\n",
        "    \"aclImdb_v1.tar.gz\", url, untar=True, cache_dir=path, cache_subdir=\"\"\n",
        ")\n",
        "\n",
        "dataset_dir = os.path.join(os.path.dirname(dataset), \"aclImdb\")\n",
        "\n",
        "train_dir = os.path.join(dataset_dir, \"train\")\n",
        "\n",
        "# remove unused folders to make it easier to load the data\n",
        "remove_dir = os.path.join(train_dir, \"unsup\")\n",
        "shutil.rmtree(remove_dir)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6IwI_2bcIeX8",
        "tags": []
      },
      "outputs": [],
      "source": [
        "AUTOTUNE = tf.data.AUTOTUNE\n",
        "batch_size = 32\n",
        "seed = 42\n",
        "\n",
        "raw_train_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
        "    path + \"aclImdb/train\",\n",
        "    batch_size=batch_size,\n",
        "    validation_split=0.2,\n",
        "    subset=\"training\",\n",
        "    seed=seed,\n",
        ")\n",
        "\n",
        "class_names = raw_train_ds.class_names\n",
        "train_ds = raw_train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n",
        "\n",
        "val_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
        "    path + \"aclImdb/train\",\n",
        "    batch_size=batch_size,\n",
        "    validation_split=0.2,\n",
        "    subset=\"validation\",\n",
        "    seed=seed,\n",
        ")\n",
        "\n",
        "val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)\n",
        "\n",
        "test_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
        "    path + \"aclImdb/test\", batch_size=batch_size\n",
        ")\n",
        "\n",
        "test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JuxDkcvVIoev",
        "tags": []
      },
      "outputs": [],
      "source": [
        "for text_batch, label_batch in train_ds.take(1):\n",
        "    for i in range(3):\n",
        "        print(f\"Review: {text_batch.numpy()[i]}\")\n",
        "        label = label_batch.numpy()[i]\n",
        "        print(f\"Label : {label} ({class_names[label]})\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "y8_ctG55-uTX",
        "tags": []
      },
      "outputs": [],
      "source": [
        "# defining the URL of the smallBERT model to use\n",
        "tfhub_handle_encoder = (\n",
        "    \"https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1\"\n",
        ")\n",
        "\n",
        "# defining the corresponding preprocessing model for the BERT model above\n",
        "tfhub_handle_preprocess = (\n",
        "    \"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\"\n",
        ")\n",
        "\n",
        "print(f\"BERT model selected           : {tfhub_handle_encoder}\")\n",
        "print(f\"Preprocess model auto-selected: {tfhub_handle_preprocess}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5soEN-YnsYm9"
      },
      "outputs": [],
      "source": [
        "bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r9-zCzJpnuwS",
        "tags": []
      },
      "outputs": [],
      "source": [
        "text_test = [\"this is such an amazing movie!\"]\n",
        "text_preprocessed = bert_preprocess_model(text_test)\n",
        "\n",
        "# This print box will help you inspect the keys in the pre-processed dictionary\n",
        "print(f\"Keys       : {list(text_preprocessed.keys())}\")\n",
        "\n",
        "# 1. input_word_ids is the ids for the words in the tokenized sentence\n",
        "print(f'Shape      : {text_preprocessed[\"input_word_ids\"].shape}')\n",
        "print(f'Word Ids   : {text_preprocessed[\"input_word_ids\"][0, :12]}')\n",
        "\n",
        "# 2. input_mask is the tokens which we are masking (masked language model)\n",
        "print(f'Input Mask : {text_preprocessed[\"input_mask\"][0, :12]}')\n",
        "\n",
        "# 3. input_type_ids is the sentence id of the input sentence.\n",
        "print(f'Type Ids   : {text_preprocessed[\"input_type_ids\"][0, :12]}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tXxYpK8ixL34",
        "tags": []
      },
      "outputs": [],
      "source": [
        "bert_model = hub.KerasLayer(tfhub_handle_encoder)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_OoF9mebuSZc",
        "tags": []
      },
      "outputs": [],
      "source": [
        "bert_results = bert_model(text_preprocessed)\n",
        "\n",
        "print(f\"Loaded BERT: {tfhub_handle_encoder}\")\n",
        "print(f'Pooled Outputs Shape:{bert_results[\"pooled_output\"].shape}')\n",
        "print(f'Pooled Outputs Values:{bert_results[\"pooled_output\"][0, :12]}')\n",
        "print(f'Sequence Outputs Shape:{bert_results[\"sequence_output\"].shape}')\n",
        "print(f'Sequence Outputs Values:{bert_results[\"sequence_output\"][0, :12]}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aksj743St9ga",
        "tags": []
      },
      "outputs": [],
      "source": [
        "def build_classifier_model(dropout_rate=0.1):\n",
        "    text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name=\"text\")\n",
        "    preprocessing_layer = hub.KerasLayer(\n",
        "        tfhub_handle_preprocess, name=\"preprocessing\"\n",
        "    )\n",
        "    encoder_inputs = preprocessing_layer(text_input)\n",
        "    encoder = hub.KerasLayer(\n",
        "        tfhub_handle_encoder, trainable=True, name=\"BERT_encoder\"\n",
        "    )\n",
        "    outputs = encoder(encoder_inputs)\n",
        "    net = outputs[\"pooled_output\"]\n",
        "    net = tf.keras.layers.Dropout(dropout_rate)(net)\n",
        "    net = tf.keras.layers.Dense(1, activation=\"sigmoid\", name=\"classifier\")(net)\n",
        "    return tf.keras.Model(text_input, net)\n",
        "\n",
        "\n",
        "# Let's check that the model runs with the output of the preprocessing model.\n",
        "dropout_rate = 0.15\n",
        "classifier_model = build_classifier_model(dropout_rate)\n",
        "bert_raw_result = classifier_model(tf.constant(text_test))\n",
        "print(bert_raw_result)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0EmzyHZXKIpm",
        "tags": []
      },
      "outputs": [],
      "source": [
        "tf.keras.utils.plot_model(classifier_model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OWPOZE-L3AgE",
        "tags": []
      },
      "outputs": [],
      "source": [
        "loss = tf.keras.losses.BinaryCrossentropy()\n",
        "metrics = tf.metrics.BinaryAccuracy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P9eP2y9dbw32",
        "tags": []
      },
      "outputs": [],
      "source": [
        "epochs = 5\n",
        "steps_per_epoch = tf.data.experimental.cardinality(train_ds).numpy()\n",
        "num_train_steps = steps_per_epoch * epochs\n",
        "num_warmup_steps = int(0.1 * num_train_steps)\n",
        "\n",
        "init_lr = 3e-5\n",
        "optimizer = optimization.create_optimizer(\n",
        "    init_lr=init_lr,\n",
        "    num_train_steps=num_train_steps,\n",
        "    num_warmup_steps=num_warmup_steps,\n",
        "    optimizer_type=\"adamw\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-7GPDhR98jsD",
        "tags": []
      },
      "outputs": [],
      "source": [
        "classifier_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HtfDFAnN_Neu"
      },
      "outputs": [],
      "source": [
        "print(f\"Training model with {tfhub_handle_encoder}\")\n",
        "history = classifier_model.fit(\n",
        "    x=train_ds, validation_data=val_ds, epochs=epochs\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "slqB-urBV9sP"
      },
      "outputs": [],
      "source": [
        "loss, accuracy = classifier_model.evaluate(test_ds)\n",
        "\n",
        "print(f\"Loss: {loss}\")\n",
        "print(f\"Accuracy: {accuracy}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fiythcODf0xo"
      },
      "outputs": [],
      "source": [
        "history_dict = history.history\n",
        "print(history_dict.keys())\n",
        "\n",
        "acc = history_dict[\"binary_accuracy\"]\n",
        "val_acc = history_dict[\"val_binary_accuracy\"]\n",
        "loss = history_dict[\"loss\"]\n",
        "val_loss = history_dict[\"val_loss\"]\n",
        "\n",
        "epochs = range(1, len(acc) + 1)\n",
        "fig = plt.figure(figsize=(10, 6))\n",
        "fig.tight_layout()\n",
        "\n",
        "plt.subplot(2, 1, 1)\n",
        "# \"bo\" is for \"blue dot\"\n",
        "plt.plot(epochs, loss, \"r\", label=\"Training loss\")\n",
        "# b is for \"solid blue line\"\n",
        "plt.plot(epochs, val_loss, \"b\", label=\"Validation loss\")\n",
        "plt.title(\"Training and validation loss\")\n",
        "# plt.xlabel('Epochs')\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "\n",
        "plt.subplot(2, 1, 2)\n",
        "plt.plot(epochs, acc, \"r\", label=\"Training acc\")\n",
        "plt.plot(epochs, val_acc, \"b\", label=\"Validation acc\")\n",
        "plt.title(\"Training and validation accuracy\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.legend(loc=\"lower right\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ShcvqJAgVera"
      },
      "outputs": [],
      "source": [
        "dataset_name = \"imdb\"\n",
        "saved_model_path = \"./{}_bert\".format(dataset_name.replace(\"/\", \"_\"))\n",
        "TIMESTAMP = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
        "\n",
        "EXPORT_PATH = os.path.join(saved_model_path, TIMESTAMP)\n",
        "\n",
        "classifier_model.save(EXPORT_PATH, include_optimizer=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gUEWVskZjEF0"
      },
      "outputs": [],
      "source": [
        "reloaded_model = tf.saved_model.load(EXPORT_PATH)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VBWzH6exlCPS"
      },
      "outputs": [],
      "source": [
        "def print_my_examples(inputs, results):\n",
        "    result_for_printing = [\n",
        "        f\"input: {inputs[i]:<30} : score: {results[i][0]:.6f}\"\n",
        "        for i in range(len(inputs))\n",
        "    ]\n",
        "    print(*result_for_printing, sep=\"\\n\")\n",
        "    print()\n",
        "\n",
        "\n",
        "examples = [\n",
        "    \"this is such an amazing movie!\",  # this is the same sentence tried earlier\n",
        "    \"The movie was great!\",\n",
        "    \"The movie was meh.\",\n",
        "    \"The movie was okish.\",\n",
        "    \"The movie was terrible...\",\n",
        "]\n",
        "\n",
        "reloaded_results = reloaded_model(tf.constant(examples))\n",
        "original_results = classifier_model(tf.constant(examples))\n",
        "\n",
        "print(\"Results from the saved model:\")\n",
        "print_my_examples(examples, reloaded_results)\n",
        "print(\"Results from the model in memory:\")\n",
        "print_my_examples(examples, original_results)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0FdVD3973S-O"
      },
      "outputs": [],
      "source": [
        "serving_results = reloaded_model.signatures[\"serving_default\"](\n",
        "    tf.constant(examples)\n",
        ")\n",
        "\n",
        "serving_results = serving_results[\"classifier\"]\n",
        "\n",
        "print_my_examples(examples, serving_results)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "92lStRlFsYnS"
      },
      "outputs": [],
      "source": [
        "!saved_model_cli show \\\n",
        "    --tag_set serve \\\n",
        "    --signature_def serving_default \\\n",
        "    --dir {EXPORT_PATH}\n",
        "\n",
        "!find {EXPORT_PATH}\n",
        "os.environ['EXPORT_PATH'] = EXPORT_PATH"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yXRex98RsYnS"
      },
      "outputs": [],
      "source": [
        "TIMESTAMP = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
        "PROJECT = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
        "PROJECT = PROJECT[0]\n",
        "BUCKET = PROJECT\n",
        "REGION = \"us-central1\"\n",
        "MODEL_DISPLAYNAME = f\"classification-bert-{TIMESTAMP}\"\n",
        "\n",
        "print(f\"MODEL_DISPLAYNAME: {MODEL_DISPLAYNAME}\")\n",
        "\n",
        "# from https://cloud.google.com/vertex-ai/docs/predictions/pre-built-containers\n",
        "SERVING_CONTAINER_IMAGE_URI = (\n",
        "    \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-12:latest\"\n",
        ")\n",
        "\n",
        "os.environ[\"BUCKET\"] = BUCKET\n",
        "os.environ[\"REGION\"] = REGION"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iDhI8p3tsYnS"
      },
      "outputs": [],
      "source": [
        "%%bash\n",
        "# Create GCS bucket if it doesn't exist already...\n",
        "exists=$(gsutil ls -d | grep -w gs://${BUCKET}/)\n",
        "\n",
        "if [ -n \"$exists\" ]; then\n",
        "    echo -e \"Bucket exists, let's not recreate it.\"\n",
        "else\n",
        "    echo \"Creating a new GCS bucket.\"\n",
        "    gsutil mb -l ${REGION} gs://${BUCKET}\n",
        "    echo \"\\nHere are your current buckets:\"\n",
        "    gsutil ls\n",
        "fi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1FDSamRssYnT"
      },
      "outputs": [],
      "source": [
        "!gsutil cp -r $EXPORT_PATH gs://$BUCKET/$MODEL_DISPLAYNAME"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nylNoERnsYnT"
      },
      "outputs": [],
      "source": [
        "uploaded_model = aiplatform.Model.upload(\n",
        "    display_name=MODEL_DISPLAYNAME,\n",
        "    artifact_uri=f\"gs://{BUCKET}/{MODEL_DISPLAYNAME}\",\n",
        "    serving_container_image_uri=SERVING_CONTAINER_IMAGE_URI,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u2-dYhV4sYnU"
      },
      "outputs": [],
      "source": [
        "MACHINE_TYPE = \"n1-standard-4\"\n",
        "\n",
        "endpoint = uploaded_model.deploy(\n",
        "    machine_type=MACHINE_TYPE,\n",
        "    accelerator_type=None,\n",
        "    accelerator_count=None,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "86W2864csYnV"
      },
      "outputs": [],
      "source": [
        "instances = [\n",
        "    {\"text\": [\"I loved the movie and highly recomment it\"]},\n",
        "    {\"text\": [\"It was an okay movie in my opinion\"]},\n",
        "    {\"text\": [\"I hated the movie\"]},\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-P5PicShsYnV"
      },
      "outputs": [],
      "source": [
        "response = endpoint.predict(instances=instances)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KgLcwcbhsYnV"
      },
      "outputs": [],
      "source": [
        "print(\" prediction:\", response.predictions)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "toc_visible": true,
      "provenance": []
    },
    "environment": {
      "kernel": "python3",
      "name": "tf2-gpu.2-12.m109",
      "type": "gcloud",
      "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-12:m109"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.11"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}