ensemble_deep_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LEVtLR_wK3o5"
   },
   "source": [
    "# library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "F8nZKdPGogQD"
   },
   "outputs": [],
   "source": [
    "# Import necessary libraries\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.applications import ResNet50, ResNet152, VGG16\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from tensorflow.keras.models import Model, load_model\n",
    "from tensorflow.keras.layers import Input, Average\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "import random\n",
    "import time\n",
    "opt = Adam(learning_rate = 0.0001)\n",
    "model_path = './model_224_32_50'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\PC\\AppData\\Local\\Temp\\ipykernel_3564\\337460670.py:1: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.config.list_physical_devices('GPU')` instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.test.is_gpu_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.config.list_physical_devices('GPU')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PnAcR1k3K92i",
    "tags": []
   },
   "source": [
    "# Data Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-IaaxwProiUZ",
    "outputId": "e2ac254a-bdb9-433b-f0d5-4b76fb2151f2"
   },
   "outputs": [],
   "source": [
    "# Set random seed for reproducibility\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "\n",
    "# Set the image size, batch size, and number of classes\n",
    "img_width, img_height = 224, 224\n",
    "batch_size = 32\n",
    "num_classes = 2\n",
    "\n",
    "\n",
    "# Define the paths to your training and validation data directories\n",
    "data_dir = './data_original'\n",
    "#train_data = './Brain Tumor/Training - Full'\n",
    "#test_data = './Brain Tumor/Testing - Full'\n",
    "\n",
    "# Define the data generators for training and validation\n",
    "datagen = ImageDataGenerator(rescale=1./255, validation_split=0.2)\n",
    "\n",
    "\n",
    "train_generator = datagen.flow_from_directory(\n",
    "    data_dir,\n",
    "    target_size=(img_width, img_height),\n",
    "    batch_size=batch_size,\n",
    "    class_mode='categorical',\n",
    "    subset='training'\n",
    ")\n",
    "\n",
    "valid_generator = datagen.flow_from_directory(\n",
    "    data_dir,\n",
    "    target_size=(img_width, img_height),\n",
    "    batch_size=batch_size,\n",
    "    class_mode='categorical',\n",
    "    subset='validation'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "id": "LOQiP-QfLPp5"
   },
   "outputs": [],
   "source": [
    "# Get the class names\n",
    "class_names = list(train_generator.class_indices.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "id": "2pkOvnNpLPp6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['no_tumor', 'pituitary']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZM4P1ESXLIsr",
    "tags": []
   },
   "source": [
    "# ResNet 152"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Gtu7rFJdLPpT",
    "outputId": "7592c0d4-b452-432c-d5d3-f481faf42c0c"
   },
   "outputs": [],
   "source": [
    "# Load the pre-trained ResNet50 model without the top (fully connected) layers\n",
    "base_model = ResNet152(weights='imagenet', include_top=False, input_shape=(img_width, img_height, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "id": "SoCzzqGbLPpU"
   },
   "outputs": [],
   "source": [
    "# Freeze the pre-trained layers\n",
    "for layer in base_model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "x = base_model.output\n",
    "x = tf.keras.layers.GlobalAveragePooling2D()(x)\n",
    "#x = tf.keras.layers.Dense(512, activation='relu')(x)\n",
    "x = tf.keras.layers.Dense(128, activation='relu')(x)\n",
    "x = tf.keras.layers.Dense(64, activation='relu')(x)\n",
    "x = tf.keras.layers.Dropout(0.5)(x)\n",
    "#predictions = Dense(2, activation = 'softmax')(x)\n",
    "predictions = tf.keras.layers.Dense(2,activation='sigmoid')(x)\n",
    "#dense_model = Model(inputs = base_model.inputs, outputs = predictions)\n",
    "\n",
    "model_resnet = tf.keras.models.Model(inputs=base_model.input, outputs=predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "id": "h-mVyAMmLPpV"
   },
   "outputs": [],
   "source": [
    "# Compile the model\n",
    "#model_resnet.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "model_resnet.compile(optimizer=opt, loss=tf.keras.losses.BinaryCrossentropy(), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "id": "ybBGHESetFo9"
   },
   "outputs": [],
   "source": [
    "#regularization: EARLY STOPPING\n",
    "#early_stopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5K03-aFqLPpW",
    "outputId": "b86c3392-c263-4228-b26f-834dcb958506"
   },
   "outputs": [],
   "source": [
    "# Train the model\n",
    "start_time = time.time()\n",
    "\n",
    "history_resnet = model_resnet.fit(train_generator,\n",
    "          steps_per_epoch=train_generator.n // batch_size,\n",
    "          epochs=50,\n",
    "          validation_data=valid_generator ,\n",
    "          #callbacks=[early_stopping]\n",
    "          validation_steps=valid_generator.n // batch_size\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time fro ResNet 152: 1996.7869460582733 seconds\n"
     ]
    }
   ],
   "source": [
    "end_time = time.time()\n",
    "training_time = end_time - start_time\n",
    "print(f\"Training time fro ResNet 152: {training_time} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AIQpu-wzMMDI",
    "outputId": "80ee3719-aedd-4b7c-e7ac-86b9248020fe"
   },
   "outputs": [],
   "source": [
    "model_resnet.save(model_path + '/resnet152.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "L5dx6376wZJ6"
   },
   "outputs": [],
   "source": [
    "# load model\n",
    "model_resnet = load_model(model_path + '/resnet152.h5')\n",
    "model_resnet.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Re9ByMj7sKzp",
    "tags": []
   },
   "source": [
    "# Model Evaluation Metrics\n",
    "[link](https://machinelearningmastery.com/how-to-calculate-precision-recall-f1-and-more-for-deep-learning-models/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OOX55G--LPpY",
    "outputId": "16b2a66f-a897-46a1-890a-b05c67106c06"
   },
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "valid_generator.reset()\n",
    "#predictions = model.predict(valid_generator, steps=valid_generator.n // batch_size)\n",
    "start_time = time.time()\n",
    "y_pred = model_resnet.predict(valid_generator)   #yhat_probs\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "inference_time_batch = end_time - start_time\n",
    "print(\"Inference time:\", inference_time_batch)\n",
    "\n",
    "# Get the predicted classes\n",
    "predicted_classes = np.argmax(y_pred, axis=1)    #yhat_classes\n",
    "\n",
    "\n",
    "# Get the true classes\n",
    "true_classes = valid_generator.classes\n",
    "\n",
    "\n",
    "# Calculate performance metrics\n",
    "confusion = confusion_matrix(true_classes, predicted_classes)\n",
    "tn, fp, fn, tp = confusion.ravel()\n",
    "print(\"tn: \", tn ,\"fp: \", fp,\"fn: \", fn,\"tp: \", tp)\n",
    "specificity = tn / (tn + fp)\n",
    "precision = precision_score(true_classes, predicted_classes)\n",
    "recall = recall_score(true_classes, predicted_classes)\n",
    "f1 = f1_score(true_classes, predicted_classes)\n",
    "\n",
    "# Print the performance metrics\n",
    "print('Specificity:', specificity)\n",
    "print('Precision:', precision)\n",
    "print('Recall:', recall)\n",
    "print('F1 Score:', f1)\n",
    "\n",
    "Accuracy =  (tp + tn) / ( tp + tn + fp + fn)\n",
    "print(\"Accuracy: \" , Accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(true_classes, predicted_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "cm = confusion_matrix(true_classes, predicted_classes)\n",
    "#sns.heatmap(cm, annot=True, cmap='Blues')\n",
    "cm_data = [[248, 42], [30, 297]]\n",
    "sns.heatmap(cm_data, annot=True, cmap='Blues', fmt='d', xticklabels=class_names, yticklabels=class_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DbuJSC9u5ovb",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Graphs, Matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KWfD8lRZ4WN1"
   },
   "source": [
    "**331, batch 32**\n",
    "\n",
    "Specificity: 0.0\n",
    "Precision: 0.5688816855753647\n",
    "Recall: 1.0\n",
    "F1 Score: 0.7252066115702479\n",
    "\n",
    "**331, batch 64**\n",
    "tn:  96 fp:  170 fn:  129 tp:  222\n",
    "Specificity: 0.3609022556390977\n",
    "Precision: 0.5663265306122449\n",
    "Recall: 0.6324786324786325\n",
    "F1 Score: 0.5975773889636609\n",
    "\n",
    "**331, batch 64, RELU**\n",
    "\n",
    "tn:  266 fp:  0 fn:  351 tp:  0\n",
    "Specificity: 1.0\n",
    "Precision: 0.0\n",
    "Recall: 0.0\n",
    "F1 Score: 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Yplwa04hLPp4",
    "outputId": "695e0271-a6af-46d8-afbf-71f5ea5e4faf"
   },
   "outputs": [],
   "source": [
    "loss,acc = model_resnet.evaluate(valid_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 564
    },
    "id": "Ng0o66g5LPp7",
    "outputId": "1b12514c-71cb-4128-ec79-a17eeed8880a"
   },
   "outputs": [],
   "source": [
    "# Plot the confusion matrix\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.heatmap(confusion, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.title('ResNet 152 Confusion Matrix')\n",
    "plt.savefig(model_path + '/resnet_conf_matrix.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 407
    },
    "id": "rrwsjFt-LPp8",
    "outputId": "be7657cc-72d8-4ba4-f742-1d3cbce925bf"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 4))\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.plot(history_resnet.history['loss'], label='Training Loss')\n",
    "plt.plot(history_resnet.history['val_loss'], label='Validation Loss')\n",
    "plt.title('ResNet-152')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "\n",
    "\n",
    "plt.xticks([10, 20, 30])\n",
    "plt.yticks([.2, .4, .6, .8, 1, 1.2])\n",
    "plt.savefig(model_path + '/resnet_loss.png')\n",
    "\n",
    "\n",
    "# Plot training accuracy and validation accuracy\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.plot(history_resnet.history['accuracy'], label='Training Accuracy')\n",
    "plt.plot(history_resnet.history['val_accuracy'], label='Validation Accuracy')\n",
    "plt.title('ResNet-152')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.xticks([10, 20, 30])\n",
    "plt.yticks([.2, .4, .6, .8, 1, 1.2])\n",
    "plt.savefig(model_path + '/resnet_acc.png')\n",
    "# Adjust layout and display the plot\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6p_QjCMLR3QH",
    "tags": []
   },
   "source": [
    "# DenseNet 201"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZGoYW2zzR3QI",
    "outputId": "aa7b7bc9-c275-40b2-adce-eb1dea2097f7"
   },
   "outputs": [],
   "source": [
    "# Load the pre-trained ResNet50 model without the top (fully connected) layers\n",
    "base_model = tf.keras.applications.DenseNet201(weights='imagenet', include_top=False, input_shape=(img_width, img_height, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "id": "lbnSLkMYR3QJ"
   },
   "outputs": [],
   "source": [
    "# Freeze the pre-trained layers\n",
    "for layer in base_model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "# Add custom fully connected layers for classification\n",
    "x = base_model.output\n",
    "x = tf.keras.layers.GlobalAveragePooling2D()(x)\n",
    "x = tf.keras.layers.Dense(128, activation='relu')(x)\n",
    "x = tf.keras.layers.Dense(64, activation='relu')(x)\n",
    "predictions = tf.keras.layers.Dense(num_classes, activation='sigmoid')(x)\n",
    "\n",
    "# Create the final model\n",
    "model_dense = tf.keras.models.Model(inputs=base_model.input, outputs=predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "id": "0ZM7CaRSR3QK"
   },
   "outputs": [],
   "source": [
    "# Compile the model\n",
    "model_dense.compile(optimizer=opt, loss=tf.keras.losses.BinaryCrossentropy(), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tGQO3A-bR3QK",
    "outputId": "854b19b8-419f-486e-c7a7-6dfe995884a4"
   },
   "outputs": [],
   "source": [
    "# Train the model\n",
    "start_time = time.time()\n",
    "\n",
    "history_model_dense = model_dense.fit(train_generator,\n",
    "          steps_per_epoch=train_generator.n // batch_size,\n",
    "          epochs=50,\n",
    "          validation_data=valid_generator ,\n",
    "          validation_steps=valid_generator.n // batch_size\n",
    "                                      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_time = time.time()\n",
    "training_time = end_time - start_time\n",
    "print(f\"Training time for DenseNet 201: {training_time} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BYKuztuNR3QL",
    "outputId": "dadeb08a-3bcc-4923-c3ed-45eac46c083c"
   },
   "outputs": [],
   "source": [
    "model_dense.save(model_path + '/dense201.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lQxa9uJFsmQh",
    "tags": []
   },
   "source": [
    "# Model Evaluation Metrics for DenseNet\n",
    "\n",
    "\n",
    "[link](https://machinelearningmastery.com/how-to-calculate-precision-recall-f1-and-more-for-deep-learning-models/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8vmVuxQLsmQh",
    "outputId": "7af2bddf-254e-462b-8d1f-2d145d3370c5"
   },
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "valid_generator.reset()\n",
    "#predictions = model.predict(valid_generator, steps=valid_generator.n // batch_size)\n",
    "start_time = time.time()\n",
    "predictions = model_dense.predict(valid_generator)   #yhat_probs\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "inference_time_batch = end_time - start_time\n",
    "print(\"Inference time:\", inference_time_batch)\n",
    "\n",
    "# Get the predicted classes\n",
    "predicted_classes = np.argmax(predictions, axis=1)    #yhat_classes\n",
    "\n",
    "# Get the true classes\n",
    "true_classes = valid_generator.classes\n",
    "\n",
    "\n",
    "# Calculate performance metrics\n",
    "confusion = confusion_matrix(true_classes, predicted_classes)\n",
    "tn, fp, fn, tp = confusion.ravel()\n",
    "print(\"tn: \", tn ,\"fp: \", fp,\"fn: \", fn,\"tp: \", tp)\n",
    "specificity = tn / (tn + fp)\n",
    "precision = precision_score(true_classes, predicted_classes)\n",
    "recall = recall_score(true_classes, predicted_classes)\n",
    "f1 = f1_score(true_classes, predicted_classes)\n",
    "\n",
    "# Print the performance metrics\n",
    "print('Specificity:', specificity)\n",
    "print('Precision:', precision)\n",
    "print('Recall:', recall)\n",
    "print('F1 Score:', f1)\n",
    "\n",
    "Accuracy =  (tp + tn) / ( tp + tn + fp + fn)\n",
    "print(\"Accuracy: \" , Accuracy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "P5LxbmjnR3QM",
    "outputId": "ad2fc45b-4b03-40bf-f851-0e524bcf2717"
   },
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "valid_generator.reset()\n",
    "#predictions = model.predict(valid_generator, steps=valid_generator.n // batch_size)\n",
    "predictions = model_dense.predict(valid_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "RgREX7lbR3QN"
   },
   "outputs": [],
   "source": [
    "# Get the predicted classes\n",
    "predicted_classes = np.argmax(predictions, axis=1)\n",
    "\n",
    "# Get the true classes\n",
    "true_classes = valid_generator.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OuR9F4SQR3QN",
    "outputId": "9269ad72-4149-4b16-a891-97bea8242bef"
   },
   "outputs": [],
   "source": [
    "# Calculate performance metrics\n",
    "confusion = confusion_matrix(true_classes, predicted_classes)\n",
    "tn, fp, fn, tp = confusion.ravel()\n",
    "specificity = tn / (tn + fp)\n",
    "precision = precision_score(true_classes, predicted_classes)\n",
    "recall = recall_score(true_classes, predicted_classes)\n",
    "f1 = f1_score(true_classes, predicted_classes)\n",
    "\n",
    "# Print the performance metrics\n",
    "print('Specificity:', specificity)\n",
    "print('Precision:', precision)\n",
    "print('Recall:', recall)\n",
    "print('F1 Score:', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "MWW8d5h-R3QO"
   },
   "outputs": [],
   "source": [
    "Accuracy =  (tp + tn) / ( tp + tn + fp + fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-STSztVLNPLh",
    "outputId": "16fecbc7-3a63-41f4-f463-f0bf7982faaa"
   },
   "outputs": [],
   "source": [
    "Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Px8x0cJ0R3QP",
    "outputId": "85fb093c-e50d-4bfa-87d7-6b3ceba6d0a0"
   },
   "outputs": [],
   "source": [
    "loss,acc = model_dense.evaluate(valid_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 564
    },
    "id": "sPnaXOjCR3QR",
    "outputId": "66e89e54-d21e-402a-95e1-4a7f76ca9083"
   },
   "outputs": [],
   "source": [
    "# Plot the confusion matrix\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.heatmap(confusion, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.title('DenseNet 201 Confusion Matrix')\n",
    "plt.savefig(model_path + '/densenet_conf_matrix.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 407
    },
    "id": "wtU_1JVNR3QS",
    "outputId": "3c253b20-bc3a-4537-84e5-64e212709b99"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 4))\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.plot(history_model_dense.history['loss'], label='Training Loss')\n",
    "plt.plot(history_model_dense.history['val_loss'], label='Validation Loss')\n",
    "plt.title('DenseNet-201')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.savefig(model_path + '/dense_loss.png')\n",
    "plt.xticks([10, 20, 30])\n",
    "plt.yticks([.2, .4, .6, .8, 1, 1.2])\n",
    "\n",
    "# Plot training accuracy and validation accuracy\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.plot(history_model_dense.history['accuracy'], label='Training Accuracy')\n",
    "plt.plot(history_model_dense.history['val_accuracy'], label='Validation Accuracy')\n",
    "plt.title('DenseNet-201')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.xticks([10, 20, 30])\n",
    "plt.yticks([.2, .4, .6, .8, 1, 1.2])\n",
    "plt.savefig(model_path + '/densenet_acc.png')\n",
    "# Adjust layout and display the plot\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JRHiOI89UVxC",
    "tags": []
   },
   "source": [
    "# vgg 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "id": "GIkl-0oyUdiN"
   },
   "outputs": [],
   "source": [
    "# Load the pre-trained ResNet50 model without the top (fully connected) layers\n",
    "base_model = VGG16(weights='imagenet', include_top=False, input_shape=(img_width, img_height, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "id": "Y9Y8e7nCUdiP"
   },
   "outputs": [],
   "source": [
    "# Freeze the pre-trained layers\n",
    "for layer in base_model.layers:\n",
    "    layer.trainable = False\n",
    "\n",
    "# Add custom fully connected layers for classification\n",
    "x = base_model.output\n",
    "x = tf.keras.layers.GlobalAveragePooling2D()(x)\n",
    "x = tf.keras.layers.Dense(128, activation='relu')(x)\n",
    "x = tf.keras.layers.Dense(64, activation='relu')(x)\n",
    "predictions = tf.keras.layers.Dense(num_classes, activation='sigmoid')(x)\n",
    "\n",
    "# Create the final model\n",
    "model_vgg16 = tf.keras.models.Model(inputs=base_model.input, outputs=predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "id": "xjqYRwvUUdiP"
   },
   "outputs": [],
   "source": [
    "# Compile the model\n",
    "model_vgg16.compile(optimizer=opt, loss=tf.keras.losses.BinaryCrossentropy(), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bJDCCt6UUdiQ",
    "outputId": "f66b7e4c-a685-4e8c-cb95-4c4764e8f655"
   },
   "outputs": [],
   "source": [
    "# Train the model\n",
    "start_time = time.time()\n",
    "\n",
    "history_model_vgg16 = model_vgg16.fit(train_generator,\n",
    "          steps_per_epoch=train_generator.n // batch_size,\n",
    "          epochs=50,\n",
    "          validation_data=valid_generator,\n",
    "          validation_steps=valid_generator.n // batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_time = time.time()\n",
    "training_time = end_time - start_time\n",
    "print(f\"Training time for VGG 16: {training_time} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lVxy1zq3UdiS",
    "outputId": "e1887949-54a1-4267-9192-f5ea844d4113"
   },
   "outputs": [],
   "source": [
    "model_vgg16.save(model_path + '/vgg16.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ABqNpC5LUdiT",
    "outputId": "d96ae649-b8b1-41fd-dad1-4bd562bc1bf0"
   },
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "valid_generator.reset()\n",
    "#predictions = model.predict(valid_generator, steps=valid_generator.n // batch_size)\n",
    "start_time = time.time()\n",
    "predictions = model_vgg16.predict(valid_generator)\n",
    "\n",
    "end_time = time.time()\n",
    "\n",
    "inference_time_batch = end_time - start_time\n",
    "print(\"Inference time:\", inference_time_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "-b6czVXbUdiU"
   },
   "outputs": [],
   "source": [
    "# Get the predicted classes\n",
    "predicted_classes = np.argmax(predictions, axis=1)\n",
    "\n",
    "# Get the true classes\n",
    "true_classes = valid_generator.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7UF-NLXzUdiV",
    "outputId": "a3feb7f4-93cf-41ba-9c88-497c1d0eb93a"
   },
   "outputs": [],
   "source": [
    "# Calculate performance metrics\n",
    "confusion = confusion_matrix(true_classes, predicted_classes)\n",
    "tn, fp, fn, tp = confusion.ravel()\n",
    "specificity = tn / (tn + fp)\n",
    "precision = precision_score(true_classes, predicted_classes)\n",
    "recall = recall_score(true_classes, predicted_classes)\n",
    "f1 = f1_score(true_classes, predicted_classes)\n",
    "\n",
    "# Print the performance metrics\n",
    "print('Specificity:', specificity)\n",
    "print('Precision:', precision)\n",
    "print('Recall:', recall)\n",
    "print('F1 Score:', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "id": "MqJIV_97UdiV"
   },
   "outputs": [],
   "source": [
    "Accuracy =  (tp + tn) / ( tp + tn + fp + fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AilCkOf4NfLq",
    "outputId": "47011d8a-79b5-4b3d-f2ee-452dd3e21ef7"
   },
   "outputs": [],
   "source": [
    "Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8K2by40KUdiW",
    "outputId": "c094bcd3-cf7f-4c48-86bc-cd1a45f2f3c8"
   },
   "outputs": [],
   "source": [
    "loss,acc = model_vgg16.evaluate(valid_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 564
    },
    "id": "KrE5h903Udia",
    "outputId": "97f9cac6-767c-4f5b-c258-e33ca93d32e1"
   },
   "outputs": [],
   "source": [
    "# Plot the confusion matrix\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.heatmap(confusion, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.title('VGG 16 Confusion Matrix')\n",
    "plt.savefig(model_path + '/vgg_conf_matrix.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 407
    },
    "id": "E4mnmocTUdic",
    "outputId": "d198736d-173d-474c-dbc7-7f308bfb50b7"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 4))\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.plot(history_model_vgg16.history['loss'], label='Training Loss')\n",
    "plt.plot(history_model_vgg16.history['val_loss'], label='Validation Loss')\n",
    "plt.title('VGG-16')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.xticks([10, 20, 30])\n",
    "plt.yticks([.2, .4, .6, .8, 1, 1.2])\n",
    "plt.savefig(model_path + '/vgg_loss.png')\n",
    "\n",
    "# Plot training accuracy and validation accuracy\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.plot(history_model_vgg16.history['accuracy'], label='Training Accuracy')\n",
    "plt.plot(history_model_vgg16.history['val_accuracy'], label='Validation Accuracy')\n",
    "plt.title('VGG-16')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.xticks([10, 20, 30])\n",
    "plt.yticks([.2, .4, .6, .8, 1, 1.2])\n",
    "plt.savefig(model_path + '/vgg_acc.png')\n",
    "# Adjust layout and display the plot\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "B6AIY9IrxjSg"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zBTWdMddxxF5",
    "tags": []
   },
   "source": [
    "# Model Evaluation Metrics for VGG\n",
    "[link](https://machinelearningmastery.com/how-to-calculate-precision-recall-f1-and-more-for-deep-learning-models/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lVz6CX_HxxGB",
    "outputId": "abdafee5-0746-4487-cbaf-fafa5a723c84"
   },
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "valid_generator.reset()\n",
    "#predictions = model.predict(valid_generator, steps=valid_generator.n // batch_size)\n",
    "predictions = model_vgg16.predict(valid_generator)   #yhat_probs\n",
    "\n",
    "# Get the predicted classes\n",
    "predicted_classes = np.argmax(predictions, axis=1)    #yhat_classes\n",
    "\n",
    "# Get the true classes\n",
    "true_classes = valid_generator.classes\n",
    "\n",
    "\n",
    "# Calculate performance metrics\n",
    "confusion = confusion_matrix(true_classes, predicted_classes)\n",
    "tn, fp, fn, tp = confusion.ravel()\n",
    "print(\"tn: \", tn ,\"fp: \", fp,\"fn: \", fn,\"tp: \", tp)\n",
    "specificity = tn / (tn + fp)\n",
    "precision = precision_score(true_classes, predicted_classes)\n",
    "recall = recall_score(true_classes, predicted_classes)\n",
    "f1 = f1_score(true_classes, predicted_classes)\n",
    "\n",
    "# Print the performance metrics\n",
    "print('Specificity:', specificity)\n",
    "print('Precision:', precision)\n",
    "print('Recall:', recall)\n",
    "print('F1 Score:', f1)\n",
    "\n",
    "Accuracy =  (tp + tn) / ( tp + tn + fp + fn)\n",
    "print(\"Accuracy: \" , Accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mFE32-GeO_dO"
   },
   "source": [
    "# full weight optimization section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "id": "tcAFr8PB4eUY"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n",
      "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n"
     ]
    }
   ],
   "source": [
    "m1 = load_model(model_path + '/resnet152.h5')\n",
    "\n",
    "m1 = Model(inputs = m1.inputs, outputs = m1.outputs,\n",
    "          name = 'resnet152')\n",
    "\n",
    "\n",
    "m2 = load_model(model_path + '/dense201.h5')\n",
    "m2 = Model(inputs = m2.inputs, outputs = m2.outputs,\n",
    "          name = 'densenet201')\n",
    "\n",
    "\n",
    "m3 = load_model(model_path + '/vgg16.h5')\n",
    "m3 = Model(inputs = m3.inputs, outputs = m3.outputs,\n",
    "          name = 'vgg16')\n",
    "\n",
    "models = [m1, m2, m3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "id": "qQLHAqaY4kAm"
   },
   "outputs": [],
   "source": [
    "model_input = tf.keras.Input(shape = (img_height,img_width,3))\n",
    "model_outputs = [model(model_input) for model in models]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "id": "TJjmOnhb4yde"
   },
   "outputs": [],
   "source": [
    "class WeightedAverageLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, w1, w2, w3, **kwargs):\n",
    "        super(WeightedAverageLayer, self).__init__(**kwargs)\n",
    "        self.w1 = w1\n",
    "        self.w2 = w2\n",
    "        self.w3 = w3\n",
    "\n",
    "    def call(self, inputs):\n",
    "        return self.w1 * inputs[0] + self.w2 * inputs[1] + self.w3 * inputs[2]\n",
    "    #added -------------------------------------------------------------------------------------------------------------\n",
    "    # def get_config(self, w1, w2, w3):\n",
    "    #     config = super().get_config()\n",
    "    #     config.update({\n",
    "    #         self.w1 = w1\n",
    "    #         self.w2 = w2\n",
    "    #         self.w3 = w3\n",
    "    #         })\n",
    "    #     return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "id": "8SzhLQs2Fkt7"
   },
   "outputs": [],
   "source": [
    "num = [[.5, .1, .4]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "f1vqLW6LPBMq",
    "outputId": "6005e52e-4569-4d88-9f2d-c48769693279"
   },
   "outputs": [],
   "source": [
    "\n",
    "opt = Adam(learning_rate = 0.0001)\n",
    "j = 0\n",
    "for i in num:\n",
    "    a = num[j][0]\n",
    "    b = num[j][1]\n",
    "    c = num[j][2]\n",
    "    print(a,b,c)\n",
    "    ensemble_output = WeightedAverageLayer(a,b,c)(model_outputs)\n",
    "    final_model = tf.keras.Model(inputs = model_input, outputs = ensemble_output)\n",
    "\n",
    "    final_model.compile(optimizer = opt, loss=tf.keras.losses.BinaryCrossentropy(), metrics=['accuracy'])\n",
    "\n",
    "    start_time = time.time()\n",
    "\n",
    "    history_final = final_model.fit(\n",
    "        train_generator,\n",
    "          steps_per_epoch=train_generator.n // batch_size,\n",
    "          epochs=50,\n",
    "          validation_data=valid_generator,\n",
    "          validation_steps=valid_generator.n // batch_size\n",
    "    )\n",
    "\n",
    "    end_time = time.time()\n",
    "    training_time = end_time - start_time\n",
    "\n",
    "    print(f\"Training time: {training_time} seconds\")\n",
    "\n",
    "    loss,acc = final_model.evaluate(valid_generator)\n",
    "\n",
    "    # with open('/content/drive/MyDrive/Colab Notebooks/model/original data model/weight(original).txt', 'a') as file:\n",
    "    # #for number in numbers:\n",
    "    #     # Write the number to the file, followed by a newline\n",
    "    #     file.write(str(a) + ' ' + str(b) + ' ' + str(c) + ' ' + str(loss) + ' ' + str(acc) + '\\n')\n",
    "\n",
    "    j = j+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "final_model.save(model_path + '/final.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pDJj6uKDPAKK"
   },
   "source": [
    "# Model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pIInTX-EetVX",
    "outputId": "7be05e0a-54cc-416e-cd63-155cfb6ccc41"
   },
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "valid_generator.reset()\n",
    "start_time = time.time()\n",
    "#predictions = model.predict(valid_generator, steps=valid_generator.n // batch_size)\n",
    "predictions = final_model.predict(valid_generator)\n",
    "end_time = time.time()\n",
    "\n",
    "inference_time_batch = end_time - start_time\n",
    "print(\"Inference time:\", inference_time_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rXt0BHXw7zrO",
    "outputId": "11a82058-ab0a-4547-a63f-53c864525ea5"
   },
   "outputs": [],
   "source": [
    "inference_time_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OW6q9DA8etVe"
   },
   "outputs": [],
   "source": [
    "# Get the predicted classes\n",
    "predicted_classes = np.argmax(predictions, axis=1)\n",
    "\n",
    "# Get the true classes\n",
    "true_classes = valid_generator.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QhmqFjyTetVe",
    "outputId": "6e15df73-edb0-4597-aafa-e5c56b141eb1"
   },
   "outputs": [],
   "source": [
    "# Calculate performance metrics\n",
    "confusion = confusion_matrix(true_classes, predicted_classes)\n",
    "tn, fp, fn, tp = confusion.ravel()\n",
    "specificity = tn / (tn + fp)\n",
    "precision = precision_score(true_classes, predicted_classes)\n",
    "recall = recall_score(true_classes, predicted_classes)\n",
    "f1 = f1_score(true_classes, predicted_classes)\n",
    "\n",
    "# Print the performance metrics\n",
    "print('Specificity:', specificity)\n",
    "print('Precision:', precision)\n",
    "print('Recall:', recall)\n",
    "print('F1 Score:', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "anKdqfOwetVe",
    "outputId": "85ad8c29-11aa-4264-af62-b68d8c369d76"
   },
   "outputs": [],
   "source": [
    "Accuracy =  (tp + tn) / ( tp + tn + fp + fn)\n",
    "print(Accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "K_K99LjwetVf",
    "outputId": "ea4da276-60d3-4d02-d57f-0b2a0aad7fb1"
   },
   "outputs": [],
   "source": [
    "loss,acc = final_model.evaluate(valid_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 564
    },
    "id": "nHf-MNq9etVf",
    "outputId": "084af1fe-9e2e-490d-cc38-386b4dad9f2f"
   },
   "outputs": [],
   "source": [
    "# Plot the confusion matrix\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.heatmap(confusion, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names)\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('True')\n",
    "plt.title('Confusion Matrix')\n",
    "plt.savefig(model_path + '/final_conf_matrix.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 820
    },
    "id": "a374f5a8",
    "outputId": "a5f9b094-6b1c-441f-fd93-cb7f93b3634c"
   },
   "outputs": [],
   "source": [
    "#plot loss and accuracy graph\n",
    "\n",
    "x = np.arange(1, 20)\n",
    "y = np.arange(0.0, 1.0)\n",
    "#y = np.array([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0])\n",
    "\n",
    "#Plot the Loss Curves\n",
    "plt.figure(figsize=[6,4])\n",
    "#plt.xlim(0, 20)\n",
    "#plt.ylim(0, 1)\n",
    "#plt.xticks(np.arange(2,21,2))\n",
    "#plt.yticks(np.arange(0.0, 1.0, 0.1))\n",
    "print(len(history_final.history['loss']) )\n",
    "plt.plot(range(1, len(history_final.history['loss']) + 1 ), history_final.history['loss'],'r')\n",
    "plt.plot(range(1, len(history_final.history['val_loss']) + 1), history_final.history['val_loss'],'b')\n",
    "#plt.plot(history_final.history['loss'],'r')\n",
    "#plt.plot(history_final.history['val_loss'],'b')\n",
    "plt.legend(['Training loss', 'Validation Loss'])\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.title('Ensemble Model Loss Curves')\n",
    "plt.xticks([10, 20])\n",
    "plt.yticks([.2, .4, .6, .8, 1, 1.2])\n",
    "#plt.show()\n",
    "plt.savefig(model_path + '/final_loss.png')\n",
    "\n",
    "#Plot the Accuracy Curves\n",
    "plt.figure(figsize=[6,4])\n",
    "#plt.plot(history_final.history['accuracy'],'r')\n",
    "#plt.plot(history_final.history['val_accuracy'],'b')\n",
    "plt.plot(range(1, len(history_final.history['accuracy']) + 1 ), history_final.history['accuracy'],'r')\n",
    "plt.plot(range(1, len(history_final.history['val_accuracy']) + 1), history_final.history['val_accuracy'],'b')\n",
    "plt.legend(['Training Accuracy', 'Validation Accuracy'])\n",
    "plt.xlabel('Epochs ')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.title('Ensemble Model Accuracy Curves')\n",
    "plt.xticks([10, 20])\n",
    "plt.yticks([.2, .4, .6, .8, 1, 1.2])\n",
    "#plt.show()\n",
    "plt.savefig(model_path + '/final_acc.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ecfc488c"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "k7EeEkSf4mVZ"
   },
   "source": [
    "# Model Evaluation Metrics\n",
    "[link](https://machinelearningmastery.com/how-to-calculate-precision-recall-f1-and-more-for-deep-learning-models/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6v97Zu2l4mVq",
    "outputId": "e3d4a070-7aca-468e-a6c2-a4c6b40a54e2"
   },
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "valid_generator.reset()\n",
    "#predictions = model.predict(valid_generator, steps=valid_generator.n // batch_size)\n",
    "predictions = final_model.predict(valid_generator)   #yhat_probs\n",
    "\n",
    "# Get the predicted classes\n",
    "predicted_classes = np.argmax(predictions, axis=1)    #yhat_classes\n",
    "\n",
    "# Get the true classes\n",
    "true_classes = valid_generator.classes\n",
    "\n",
    "\n",
    "# Calculate performance metrics\n",
    "confusion = confusion_matrix(true_classes, predicted_classes)\n",
    "tn, fp, fn, tp = confusion.ravel()\n",
    "print(\"tn: \", tn ,\"fp: \", fp,\"fn: \", fn,\"tp: \", tp)\n",
    "specificity = tn / (tn + fp)\n",
    "precision = precision_score(true_classes, predicted_classes)\n",
    "recall = recall_score(true_classes, predicted_classes)\n",
    "f1 = f1_score(true_classes, predicted_classes)\n",
    "\n",
    "# Print the performance metrics\n",
    "print('Specificity:', specificity)\n",
    "print('Precision:', precision)\n",
    "print('Recall:', recall)\n",
    "print('F1 Score:', f1)\n",
    "\n",
    "Accuracy =  (tp + tn) / ( tp + tn + fp + fn)\n",
    "print(\"Accuracy: \" , Accuracy)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "Jvd8eRL0LEOF",
    "c_fwiNtBB4ey"
   ],
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "toc-autonumbering": true,
  "toc-showcode": false,
  "toc-showmarkdowntxt": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}