kaggle_bottle_images.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Lion mail drive link to the model: <a href=\"https://drive.google.com/drive/folders/13LU6Z5qObxH6pxdUo9N531-eqFGlvAIV?usp=sharing\">https://drive.google.com/drive/folders/13LU6Z5qObxH6pxdUo9N531-eqFGlvAIV?usp=sharing</a></h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading folder 0\n",
      "Reading folder 1\n",
      "Reading folder 2\n",
      "Reading folder 3\n",
      "Reading folder 4\n",
      "Reading Test Images\n",
      "Training data shape:  (15000, 128, 128, 3)\n",
      "Training labels shape:  (15000,)\n",
      "Test data shape:  (3500, 128, 128, 3)\n"
     ]
    }
   ],
   "source": [
    "#Generate dataset\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "\n",
    "\n",
    "#Load Training images and labels\n",
    "train_directory = \"./data/kaggle_train_128/train_128\" #TODO: Enter path for train128 folder (hint: use os.getcwd())\n",
    "image_list=[]\n",
    "label_list=[]\n",
    "for sub_dir in os.listdir(train_directory):\n",
    "    print(\"Reading folder {}\".format(sub_dir))\n",
    "    sub_dir_name=os.path.join(train_directory,sub_dir)\n",
    "    for file in os.listdir(sub_dir_name):\n",
    "        filename = os.fsdecode(file)\n",
    "        if filename.endswith(\".jpg\") or filename.endswith(\".png\"):\n",
    "            image_list.append(np.array(Image.open(os.path.join(sub_dir_name,file))))\n",
    "            label_list.append(int(sub_dir))\n",
    "X_train=np.array(image_list)\n",
    "y_train=np.array(label_list)\n",
    "\n",
    "#Load Test images\n",
    "test_directory = \"./data/kaggle_test_128/test_128\"#TODO: Enter path for test128 folder (hint: use os.getcwd())\n",
    "test_image_list=[]\n",
    "test_df = pd.DataFrame([], columns=['Id', 'X'])\n",
    "print(\"Reading Test Images\")\n",
    "for file in os.listdir(test_directory):\n",
    "    filename = os.fsdecode(file)\n",
    "    if filename.endswith(\".jpg\") or filename.endswith(\".png\"):\n",
    "        test_df = test_df.append({\n",
    "            'Id': filename,\n",
    "            'X': np.array(Image.open(os.path.join(test_directory,file)))\n",
    "        }, ignore_index=True)\n",
    "        \n",
    "test_df['s'] = [int(x.split('.')[0]) for x in test_df['Id']]\n",
    "test_df = test_df.sort_values(by=['s'])\n",
    "test_df = test_df.drop(columns=['s'])\n",
    "X_test = np.stack(test_df['X'])\n",
    "\n",
    "\n",
    "print('Training data shape: ', X_train.shape)\n",
    "print('Training labels shape: ', y_train.shape)\n",
    "print('Test data shape: ', X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Incremental steps are used to train the model.\n",
    "\n",
    "## Step 1: transfer learning with ResNet50V2 and fine tuning with data augmentation\n",
    "\n",
    "## Step 2: transfer learning with DenseNet121 and fine tuning with data augmentation\n",
    "\n",
    "## Step 3: Concactenate both models and do a final transfer learning\n",
    "\n",
    "\n",
    "<h5>Lion mail drive link to the model: <a href=\"https://drive.google.com/drive/folders/13LU6Z5qObxH6pxdUo9N531-eqFGlvAIV?usp=sharing\">https://drive.google.com/drive/folders/13LU6Z5qObxH6pxdUo9N531-eqFGlvAIV?usp=sharing</a></h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nguye\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\client\\session.py:1761: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.compat.v1 import ConfigProto\n",
    "from tensorflow.compat.v1 import InteractiveSession\n",
    "\n",
    "#from tensorflow import ConfigProto\n",
    "#from tensorflow import InteractiveSession\n",
    "config = ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = InteractiveSession(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/device:CPU:0', '/device:GPU:0']\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "def get_available_devices():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos]\n",
    "print(get_available_devices()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# YOUR CODE HERE\n",
    "# suffle data\n",
    "shuffler = np.random.permutation(len(X_train))\n",
    "X_train = X_train[shuffler]\n",
    "y_train = y_train[shuffler]\n",
    "\n",
    "y=tf.keras.utils.to_categorical(y_train)\n",
    "# validation data and training data\n",
    "Xval=X_train[0:2000]\n",
    "yval=y[0:2000]\n",
    "Xtrain=X_train[2000:]\n",
    "ytrain=y[2000:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for future data augmentation\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "bs=64\n",
    "epc=13#not used\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1a:\n",
    "transfer learning with ResNet50V2 using only the original training data (with 13% of the data for validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model  ResNet50 pretrained\n",
    "#from tensorflow.keras.applications.inception_v3 import InceptionV3\n",
    "#base_model = InceptionV3(input_shape = (128, 128, 3), include_top = False, weights = 'imagenet')\n",
    "\n",
    "\n",
    "from tensorflow.keras.applications.resnet_v2 import ResNet50V2\n",
    "base_model = ResNet50V2( include_top=False, weights=\"imagenet\")\n",
    "preprocess_input =tf.keras.applications.resnet_v2.preprocess_input #to ensure proper input format into resnet50V2\n",
    "\n",
    "for layer in base_model.layers: #Freeze resnet50v2 layers for Transfer learning \n",
    "    layer.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.keras.models import Sequential\n",
    "from tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D,Dropout\n",
    "from tensorflow.keras import regularizers\n",
    "#input\n",
    "inputs = tf.keras.Input(shape=(128, 128, 3))\n",
    "# Preprocess for ResNet50v2layer \n",
    "x = preprocess_input(inputs)\n",
    "# ResNet50v2layer \n",
    "x = base_model(x, training=False)\n",
    "x=GlobalAveragePooling2D()(x)\n",
    "\n",
    "\n",
    "# we add dense layers so that the model can learn more complex functions and classify for better results.\n",
    "# Regularization to avoid overfitting\n",
    "x=Dense(2048,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),\n",
    "    bias_regularizer=regularizers.l2(1e-4))(x) \n",
    "x=Dropout(0.2)(x)\n",
    "x=Dense(2048,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),\n",
    "    bias_regularizer=regularizers.l2(1e-4))(x)\n",
    "x=Dropout(0.2)(x)\n",
    "x=Dense(512,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),\n",
    "    bias_regularizer=regularizers.l2(1e-4))(x)\n",
    "x=Dense(256,activation='relu')(x)\n",
    "preds=Dense(5,activation='softmax')(x)\n",
    "\n",
    "\n",
    "model=Model(inputs=inputs,outputs=preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 128, 128, 3)]     0         \n",
      "_________________________________________________________________\n",
      "tf.math.truediv (TFOpLambda) (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "tf.math.subtract (TFOpLambda (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "resnet50v2 (Functional)      (None, None, None, 2048)  23564800  \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d (Gl (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2048)              4196352   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2048)              4196352   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               1049088   \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 5)                 1285      \n",
      "=================================================================\n",
      "Total params: 33,139,205\n",
      "Trainable params: 9,574,405\n",
      "Non-trainable params: 23,564,800\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "base_learning_rate = 0.0001\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "204/204 [==============================] - 25s 111ms/step - loss: 3.5372 - acc: 0.4813 - val_loss: 3.1002 - val_acc: 0.6270\n",
      "Epoch 2/6\n",
      "204/204 [==============================] - 15s 72ms/step - loss: 2.9895 - acc: 0.6542 - val_loss: 2.8519 - val_acc: 0.6725\n",
      "Epoch 3/6\n",
      "204/204 [==============================] - 15s 72ms/step - loss: 2.7486 - acc: 0.7246 - val_loss: 2.7742 - val_acc: 0.6900\n",
      "Epoch 4/6\n",
      "204/204 [==============================] - 15s 73ms/step - loss: 2.5843 - acc: 0.7615 - val_loss: 2.7069 - val_acc: 0.6895\n",
      "Epoch 5/6\n",
      "204/204 [==============================] - 15s 73ms/step - loss: 2.4128 - acc: 0.8150 - val_loss: 2.6972 - val_acc: 0.6930\n",
      "Epoch 6/6\n",
      "204/204 [==============================] - 15s 73ms/step - loss: 2.2597 - acc: 0.8605 - val_loss: 2.6312 - val_acc: 0.7250\n"
     ]
    }
   ],
   "source": [
    "history=model.fit(x=Xtrain, y=ytrain,\n",
    "                   batch_size = bs,\n",
    "                   epochs = 6,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: model1\\assets\n"
     ]
    }
   ],
   "source": [
    "#Save step 1 for potential reuse\n",
    "model.save(\"model1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1b\n",
    "\n",
    "More transfer learning using conservative data augmentation on the same data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.1,\n",
    "        zoom_range=0.1,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=10.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.1, \n",
    "        height_shift_range = 0.1)\n",
    "        #brightness_range=(0.3, 1.4),\n",
    "        #preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(Xtrain, ytrain, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "203/203 [==============================] - 51s 234ms/step - loss: 2.6573 - acc: 0.6589 - val_loss: 2.1799 - val_acc: 0.8515\n",
      "Epoch 2/6\n",
      "203/203 [==============================] - 42s 206ms/step - loss: 2.0680 - acc: 0.8525 - val_loss: 1.8727 - val_acc: 0.9060\n",
      "Epoch 3/6\n",
      "203/203 [==============================] - 41s 203ms/step - loss: 1.8384 - acc: 0.8919 - val_loss: 1.6484 - val_acc: 0.9290\n",
      "Epoch 4/6\n",
      "203/203 [==============================] - 41s 204ms/step - loss: 1.6520 - acc: 0.9122 - val_loss: 1.7058 - val_acc: 0.8705\n",
      "Epoch 5/6\n",
      "203/203 [==============================] - 41s 204ms/step - loss: 1.4945 - acc: 0.9222 - val_loss: 1.3952 - val_acc: 0.9350\n",
      "Epoch 6/6\n",
      "203/203 [==============================] - 41s 204ms/step - loss: 1.3617 - acc: 0.9275 - val_loss: 1.3214 - val_acc: 0.9190\n",
      "INFO:tensorflow:Assets written to: model2\\assets\n"
     ]
    }
   ],
   "source": [
    "#step2 \n",
    "# start from where we ended in step 1 (same weights)\n",
    "model2 = tf.keras.models.load_model(\"model1\")\n",
    "history=model2.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs = 6,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n",
    "model2.save(\"model2\") #Save step 2 for potential reuse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1c\n",
    "\n",
    "More transfer learning using more aggressive data augmentation settings on the same data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Even more data augmentation\n",
    "def add_noise(img):\n",
    "    '''Add random noise to an image'''\n",
    "    noise = np.random.normal(0, 5, img.shape)\n",
    "    img += noise\n",
    "    np.clip(img, 0., 255.)\n",
    "    return img\n",
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.2,\n",
    "        zoom_range=0.2,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=20.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.15, \n",
    "        height_shift_range = 0.15,\n",
    "        brightness_range=(0.3, 1.4),\n",
    "        preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(Xtrain, ytrain, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "203/203 [==============================] - 67s 311ms/step - loss: 1.3224 - acc: 0.8724 - val_loss: 1.0170 - val_acc: 0.9385\n",
      "Epoch 2/2\n",
      "203/203 [==============================] - 63s 311ms/step - loss: 1.0766 - acc: 0.8920 - val_loss: 0.8853 - val_acc: 0.9445\n",
      "INFO:tensorflow:Assets written to: model_aug\\assets\n"
     ]
    }
   ],
   "source": [
    "#step3\n",
    "# start from where we ended in step 2 (same weights)\n",
    "model3= tf.keras.models.load_model(\"model2\")\n",
    "history=model3.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs = 2,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n",
    "model3.save(\"model_aug\") #Save step 3 for potential reuse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1d\n",
    "\n",
    "Fine tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fine-tuning\n",
    "# start from where we ended in step 3 (same weights)\n",
    "model4 = tf.keras.models.load_model(\"model_aug\")\n",
    "for layer in model4.layers:\n",
    "    if (layer.name=='resnet50v2'):\n",
    "        layer.trainable=True\n",
    "        for lay in layer.layers:\n",
    "            lay.trainable = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "203/203 [==============================] - 65s 305ms/step - loss: 0.9271 - acc: 0.8949 - val_loss: 0.7920 - val_acc: 0.9275\n",
      "Epoch 2/3\n",
      "203/203 [==============================] - 62s 304ms/step - loss: 0.8051 - acc: 0.9038 - val_loss: 0.6392 - val_acc: 0.9505\n",
      "Epoch 3/3\n",
      "203/203 [==============================] - 64s 314ms/step - loss: 0.7192 - acc: 0.9099 - val_loss: 0.5941 - val_acc: 0.9420\n"
     ]
    }
   ],
   "source": [
    "history=model4.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs =3,\n",
    "                   validation_data=(Xval, yval),shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: model4\\assets\n"
     ]
    }
   ],
   "source": [
    "#Save step 4 for potential reuse\n",
    "model4.save(\"model4\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1e\n",
    "More fine tuning with different hyper parameters on all the provided training data\n",
    "(Kaggle best score was around 97.8% accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start from where we ended in step 4 (same weights)\n",
    "model5 = tf.keras.models.load_model(\"model4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#full data set this time (15000 pictures)\n",
    "#Different hyperparameters\n",
    "bs=32\n",
    "train_generator = train_datagen.flow(X_train, y, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size\n",
    "base_learning_rate = 0.00005\n",
    "#need to compile because of the different learning rate\n",
    "model5.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/13\n",
      "468/468 [==============================] - 79s 131ms/step - loss: 0.5939 - acc: 0.9296\n",
      "Epoch 2/13\n",
      "468/468 [==============================] - 63s 135ms/step - loss: 0.5327 - acc: 0.9337\n",
      "Epoch 3/13\n",
      "468/468 [==============================] - 61s 131ms/step - loss: 0.4895 - acc: 0.9368\n",
      "Epoch 4/13\n",
      "468/468 [==============================] - 62s 133ms/step - loss: 0.4668 - acc: 0.9391\n",
      "Epoch 5/13\n",
      "468/468 [==============================] - 62s 133ms/step - loss: 0.4269 - acc: 0.9421\n",
      "Epoch 6/13\n",
      "468/468 [==============================] - 63s 134ms/step - loss: 0.4028 - acc: 0.9417\n",
      "Epoch 7/13\n",
      "468/468 [==============================] - 63s 134ms/step - loss: 0.3704 - acc: 0.9439\n",
      "Epoch 8/13\n",
      "468/468 [==============================] - 63s 134ms/step - loss: 0.3716 - acc: 0.9420\n",
      "Epoch 9/13\n",
      "468/468 [==============================] - 62s 133ms/step - loss: 0.3271 - acc: 0.9517\n",
      "Epoch 10/13\n",
      "468/468 [==============================] - 63s 134ms/step - loss: 0.3051 - acc: 0.9543\n",
      "Epoch 11/13\n",
      "468/468 [==============================] - 63s 135ms/step - loss: 0.3010 - acc: 0.9519\n",
      "Epoch 12/13\n",
      "468/468 [==============================] - 63s 134ms/step - loss: 0.2965 - acc: 0.9480\n",
      "Epoch 13/13\n",
      "423/468 [==========================>...] - ETA: 6s - loss: 0.2718 - acc: 0.9565"
     ]
    }
   ],
   "source": [
    "history=model5.fit(train_generator,steps_per_epoch = step_size_train,epochs =13,shuffle=True)\n",
    "model5.save(\"model5\") #Save step 5 for potential reuse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1f\n",
    "More fine tuning with even more aggressive data augmentation, with different hyper parameters for stability.\n",
    "\n",
    "Accuracy with temporary kaggle estimation: 98.204% accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs=128\n",
    "def add_noise(img):\n",
    "    '''Add random noise to an image'''\n",
    "    noise = np.random.normal(0, 7, img.shape)\n",
    "    img += noise\n",
    "    np.clip(img, 0., 255.)\n",
    "    return img\n",
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.2,\n",
    "        zoom_range=0.2,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=35.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.2, \n",
    "        height_shift_range = 0.2,\n",
    "        brightness_range=(0.3, 1.3),\n",
    "        preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(Xtrain, ytrain, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "101/101 [==============================] - 78s 602ms/step - loss: 0.3860 - acc: 0.9125\n",
      "Epoch 2/3\n",
      "101/101 [==============================] - 52s 507ms/step - loss: 0.3203 - acc: 0.9315\n",
      "Epoch 3/3\n",
      "101/101 [==============================] - 52s 509ms/step - loss: 0.2892 - acc: 0.9417\n",
      "INFO:tensorflow:Assets written to: model6\\assets\n"
     ]
    }
   ],
   "source": [
    "model6 = tf.keras.models.load_model(\"model5\") # start from where we ended in step 5 (same weights)\n",
    "base_learning_rate = 0.00005\n",
    "model6.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "history=model6.fit(train_generator,steps_per_epoch = step_size_train,epochs =3,shuffle=True)\n",
    "model6.save(\"model6\") #Save step 6 for potential reuse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2a:\n",
    "transfer learning with DenseNet121 using only the original training data (with 13% of the data for validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# YOUR CODE HERE\n",
    "# suffle data\n",
    "shuffler = np.random.permutation(len(X_train))\n",
    "X_train = X_train[shuffler]\n",
    "y_train = y_train[shuffler]\n",
    "\n",
    "y=tf.keras.utils.to_categorical(y_train)\n",
    "# validation data and training data\n",
    "Xval=X_train[0:2000]\n",
    "yval=y[0:2000]\n",
    "Xtrain=X_train[2000:]\n",
    "ytrain=y[2000:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for future data augmentation\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "bs=64\n",
    "epc=13#not used\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = tf.keras.applications.densenet.DenseNet121( include_top=False, weights=\"imagenet\")\n",
    "preprocess_input =tf.keras.applications.densenet.preprocess_input #to ensure proper input format into resnet50V2\n",
    "\n",
    "for layer in base_model.layers: #Freeze resnet50v2 layers for Transfer learning \n",
    "    layer.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.keras.models import Sequential\n",
    "from tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D,Dropout\n",
    "from tensorflow.keras import regularizers\n",
    "#input\n",
    "inputs = tf.keras.Input(shape=(128, 128, 3))\n",
    "# Preprocess for ResNet50v2layer \n",
    "x = preprocess_input(inputs)\n",
    "# ResNet50v2layer \n",
    "x = base_model(x, training=False)\n",
    "x=GlobalAveragePooling2D()(x)\n",
    "\n",
    "\n",
    "# we add dense layers so that the model can learn more complex functions and classify for better results.\n",
    "# Regularization to avoid overfitting\n",
    "x=Dense(1024,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),\n",
    "    bias_regularizer=regularizers.l2(1e-4))(x) \n",
    "x=Dropout(0.2)(x)\n",
    "x=Dense(2048,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),\n",
    "    bias_regularizer=regularizers.l2(1e-4))(x)\n",
    "x=Dropout(0.2)(x)\n",
    "x=Dense(512,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),\n",
    "    bias_regularizer=regularizers.l2(1e-4))(x)\n",
    "x=Dense(256,activation='relu')(x)\n",
    "preds=Dense(5,activation='softmax')(x)\n",
    "\n",
    "\n",
    "model=Model(inputs=inputs,outputs=preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 128, 128, 3)]     0         \n",
      "_________________________________________________________________\n",
      "tf.math.truediv (TFOpLambda) (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "tf.nn.bias_add (TFOpLambda)  (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "tf.math.truediv_1 (TFOpLambd (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "densenet121 (Functional)     (None, None, None, 1024)  7037504   \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d (Gl (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1024)              1049600   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2048)              2099200   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               1049088   \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 5)                 1285      \n",
      "=================================================================\n",
      "Total params: 11,368,005\n",
      "Trainable params: 4,330,501\n",
      "Non-trainable params: 7,037,504\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "base_learning_rate = 0.0001\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "204/204 [==============================] - 66s 216ms/step - loss: 2.6331 - acc: 0.4175 - val_loss: 2.1189 - val_acc: 0.6470\n",
      "Epoch 2/6\n",
      "204/204 [==============================] - 16s 77ms/step - loss: 2.0869 - acc: 0.6315 - val_loss: 1.9583 - val_acc: 0.6765\n",
      "Epoch 3/6\n",
      "204/204 [==============================] - 16s 77ms/step - loss: 1.9025 - acc: 0.6883 - val_loss: 1.8449 - val_acc: 0.6985\n",
      "Epoch 4/6\n",
      "204/204 [==============================] - 16s 77ms/step - loss: 1.7572 - acc: 0.7383 - val_loss: 1.7895 - val_acc: 0.7165\n",
      "Epoch 5/6\n",
      "204/204 [==============================] - 16s 77ms/step - loss: 1.6512 - acc: 0.7651 - val_loss: 1.7906 - val_acc: 0.7110\n",
      "Epoch 6/6\n",
      "204/204 [==============================] - 16s 77ms/step - loss: 1.5760 - acc: 0.7858 - val_loss: 1.7468 - val_acc: 0.7270\n"
     ]
    }
   ],
   "source": [
    "history=model.fit(x=Xtrain, y=ytrain,\n",
    "                   batch_size = bs,\n",
    "                   epochs = 6,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./densenet_models/model1\\assets\n"
     ]
    }
   ],
   "source": [
    "#Save step 1 for potential reuse\n",
    "model.save(\"./densenet_models/model1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2b\n",
    "\n",
    "More transfer learning using conservative data augmentation on the same data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.1,\n",
    "        zoom_range=0.1,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=10.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.1, \n",
    "        height_shift_range = 0.1)\n",
    "        #brightness_range=(0.3, 1.4),\n",
    "        #preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(Xtrain, ytrain, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#step2 \n",
    "# start from where we ended in step 1 (same weights)\n",
    "model2 = tf.keras.models.load_model(\"./densenet_models/model1\")\n",
    "history=model2.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs = 6,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n",
    "model2.save(\"./densenet_models/model2\") #Save step 2 for potential reuse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2c\n",
    "\n",
    "More transfer learning using more aggressive data augmentation settings on the same data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Even more data augmentation\n",
    "def add_noise(img):\n",
    "    '''Add random noise to an image'''\n",
    "    noise = np.random.normal(0, 5, img.shape)\n",
    "    img += noise\n",
    "    np.clip(img, 0., 255.)\n",
    "    return img\n",
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.2,\n",
    "        zoom_range=0.2,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=20.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.15, \n",
    "        height_shift_range = 0.15,\n",
    "        brightness_range=(0.3, 1.4),\n",
    "        preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(Xtrain, ytrain, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "203/203 [==============================] - 130s 484ms/step - loss: 1.0024 - acc: 0.9106 - val_loss: 0.8209 - val_acc: 0.9625\n",
      "INFO:tensorflow:Assets written to: ./densenet_models/model_aug\\assets\n"
     ]
    }
   ],
   "source": [
    "#step3\n",
    "# start from where we ended in step 2 (same weights)\n",
    "model3= tf.keras.models.load_model(\"./densenet_models/model2\")\n",
    "history=model3.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs = 1,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n",
    "model3.save(\"./densenet_models/model_aug\") #Save step 3 for potential reuse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2d\n",
    "\n",
    "Fine tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fine-tuning\n",
    "# start from where we ended in step 3 (same weights)\n",
    "model4 = tf.keras.models.load_model(\"./densenet_models/model_aug\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_2\n",
      "tf.math.truediv\n",
      "tf.nn.bias_add\n",
      "tf.math.truediv_1\n",
      "densenet121\n",
      "global_average_pooling2d\n",
      "dense\n",
      "dropout\n",
      "dense_1\n",
      "dropout_1\n",
      "dense_2\n",
      "dense_3\n",
      "dense_4\n",
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 128, 128, 3)]     0         \n",
      "_________________________________________________________________\n",
      "tf.math.truediv (TFOpLambda) (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "tf.nn.bias_add (TFOpLambda)  (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "tf.math.truediv_1 (TFOpLambd (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "densenet121 (Functional)     (None, None, None, 1024)  7037504   \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d (Gl (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1024)              1049600   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2048)              2099200   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               1049088   \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 5)                 1285      \n",
      "=================================================================\n",
      "Total params: 11,368,005\n",
      "Trainable params: 11,284,357\n",
      "Non-trainable params: 83,648\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "for layer in model4.layers:\n",
    "    print(layer.name)\n",
    "    if (layer.name=='densenet121'):\n",
    "        layer.trainable=True\n",
    "        for lay in layer.layers:\n",
    "            lay.trainable = True\n",
    "model4.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "203/203 [==============================] - 130s 486ms/step - loss: 0.8734 - acc: 0.9243 - val_loss: 0.7060 - val_acc: 0.9710\n",
      "Epoch 2/2\n",
      "203/203 [==============================] - 57s 279ms/step - loss: 0.7795 - acc: 0.9339 - val_loss: 0.6564 - val_acc: 0.9720\n"
     ]
    }
   ],
   "source": [
    "history=model4.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs =2,\n",
    "                   validation_data=(Xval, yval),shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./densenet_models/model4\\assets\n"
     ]
    }
   ],
   "source": [
    "#Save step 4 for potential reuse\n",
    "model4.save(\"./densenet_models/model4\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2e\n",
    "More fine tuning with even more aggressive data augmentation, with different hyper parameters for stability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs=80\n",
    "def add_noise(img):\n",
    "    '''Add random noise to an image'''\n",
    "    noise = np.random.normal(0, 7, img.shape)\n",
    "    img += noise\n",
    "    np.clip(img, 0., 255.)\n",
    "    return img\n",
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.2,\n",
    "        zoom_range=0.2,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=35.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.2, \n",
    "        height_shift_range = 0.2,\n",
    "        brightness_range=(0.3, 1.3),\n",
    "        preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(Xtrain, ytrain, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "162/162 [==============================] - 108s 472ms/step - loss: 0.8466 - acc: 0.9006\n",
      "Epoch 2/3\n",
      "162/162 [==============================] - 54s 332ms/step - loss: 0.7428 - acc: 0.9212\n",
      "Epoch 3/3\n",
      "162/162 [==============================] - 54s 332ms/step - loss: 0.6911 - acc: 0.9304\n",
      "INFO:tensorflow:Assets written to: ./densenet_models/model6\\assets\n"
     ]
    }
   ],
   "source": [
    "model6 = tf.keras.models.load_model(\"./densenet_models/model4\") # start from where we ended in step 5 (same weights)\n",
    "base_learning_rate = 0.00005\n",
    "model6.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "history=model6.fit(train_generator,steps_per_epoch = step_size_train,epochs =3,shuffle=True)\n",
    "model6.save(\"./densenet_models/model6\") #Save step 6 for potential reuse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.compat.v1 import ConfigProto\n",
    "from tensorflow.compat.v1 import InteractiveSession\n",
    "\n",
    "#from tensorflow import ConfigProto\n",
    "#from tensorflow import InteractiveSession\n",
    "config = ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = InteractiveSession(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/device:CPU:0']\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "def get_available_devices():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos]\n",
    "print(get_available_devices()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# YOUR CODE HERE\n",
    "# suffle data\n",
    "shuffler = np.random.permutation(len(X_train))\n",
    "X_train = X_train[shuffler]\n",
    "y_train = y_train[shuffler]\n",
    "\n",
    "y=tf.keras.utils.to_categorical(y_train)\n",
    "# validation data and training data\n",
    "Xval=X_train[0:2000]\n",
    "yval=y[0:2000]\n",
    "Xtrain=X_train[2000:]\n",
    "ytrain=y[2000:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for future data augmentation\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "bs=64\n",
    "epc=13#not used\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3a:\n",
    "Concatenate both previous models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# YOUR CODE HERE\n",
    "# suffle data\n",
    "shuffler = np.random.permutation(len(X_train))\n",
    "X_train = X_train[shuffler]\n",
    "y_train = y_train[shuffler]\n",
    "\n",
    "y=tf.keras.utils.to_categorical(y_train)\n",
    "# validation data and training data\n",
    "Xval=X_train[0:2000]#not used here, all labeled data is used for training \n",
    "yval=y[0:2000]#not used here, all labeled data is used for training\n",
    "Xtrain=X_train[2000:]#not used here, all labeled data is used for training\n",
    "ytrain=y[2000:]#not used here, all labeled data is used for training \n",
    "#Because we use such small number of epoch and very small learning rate, overfitting is less likely to happen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for future data augmentation\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "bs=64\n",
    "epc=13#not used\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = tf.keras.models.load_model(\"./densenet_models/model6\")\n",
    "dense = tf.keras.models.load_model(\"model6\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Dense at 0x193f00fcc48>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res._layers.pop()#remove top layer\n",
    "dense._layers.pop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 128, 128, 3)]     0         \n",
      "_________________________________________________________________\n",
      "tf.math.truediv (TFOpLambda) (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "tf.nn.bias_add (TFOpLambda)  (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "tf.math.truediv_1 (TFOpLambd (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "densenet121 (Functional)     (None, None, None, 1024)  7037504   \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d (Gl (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1024)              1049600   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2048)              2099200   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 2048)              0         \n",
      "=================================================================\n",
      "Total params: 10,186,304\n",
      "Trainable params: 0\n",
      "Non-trainable params: 10,186,304\n",
      "_________________________________________________________________\n",
      "Model: \"model_10\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_8 (InputLayer)         [(None, 128, 128, 3)]     0         \n",
      "_________________________________________________________________\n",
      "tf.math.truediv_5 (TFOpLambd (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "tf.math.subtract_5 (TFOpLamb (None, 128, 128, 3)       0         \n",
      "_________________________________________________________________\n",
      "resnet50v2 (Functional)      (None, None, None, 2048)  23564800  \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_4 ( (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 2048)              4196352   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 2048)              4196352   \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 2048)              0         \n",
      "=================================================================\n",
      "Total params: 31,957,504\n",
      "Trainable params: 0\n",
      "Non-trainable params: 31,957,504\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D,Dropout\n",
    "from tensorflow.keras import regularizers\n",
    "for layer in dense.layers:\n",
    "    layer.trainable=False\n",
    "for layer in res.layers:\n",
    "    layer.trainable=False\n",
    "res._name=\"res_kaggle\"\n",
    "dense._name=\"dense_kaggle\"\n",
    "\n",
    "r = Model(res.input, res.layers[-3].output)\n",
    "r.summary()\n",
    "d = Model(dense.input, dense.layers[-3].output)\n",
    "d.summary()\n",
    "r._name=\"resnet_50V2_trained\"\n",
    "d._name=\"densenet_121_trained\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#input\n",
    "inputs = tf.keras.Input(shape=(128, 128, 3))\n",
    "x = r(inputs, training=False)\n",
    "y = d(inputs, training=False)\n",
    "combined =  tf.keras.layers.Concatenate()([x, y])\n",
    "z=Dense(2048,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),\n",
    "    bias_regularizer=regularizers.l2(1e-4))(combined)\n",
    "z=Dropout(0.2)(z)\n",
    "z=Dense(512,activation='relu',kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4),\n",
    "    bias_regularizer=regularizers.l2(1e-4))(z)\n",
    "z=Dense(256,activation='relu')(z)\n",
    "preds=Dense(5,activation='softmax')(z)\n",
    "\n",
    "model=Model(inputs=inputs,outputs=preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_11\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_4 (InputLayer)            [(None, 128, 128, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "resnet_50V2_trained (Functional (None, 2048)         10186304    input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "densenet_121_trained (Functiona (None, 2048)         31957504    input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 4096)         0           resnet_50V2_trained[0][0]        \n",
      "                                                                 densenet_121_trained[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 2048)         8390656     concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 2048)         0           dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 512)          1049088     dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 256)          131328      dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 5)            1285        dense_14[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 51,716,165\n",
      "Trainable params: 9,572,357\n",
      "Non-trainable params: 42,143,808\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "base_learning_rate = 0.00005\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./concat_models/model0\\assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"./concat_models/model0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAANQCAYAAABpemflAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdfXAT17k/8K94aQJmxi5J7RZak6QphLyZhntTM/m1GV56M6RdQTr4RQQDDbYiQcB08NzbMvIwjBna6cgtBAJGNgQiQLLNkGBNkt4Jdi+ZNnbS0lhpaIqTmNoFJ1IziQQpBQLs7w96FkleyStb8url+5nRgHZXZ599sfbo2bPnGGRZlkFERERERERElBitY/SOgIiIiIiIiIgyC5MNRERERERERJRQTDYQERERERERUUIx2UBERERERERECTVO7wCIiJLhV7/6FTo7O/UOg4iISNHa2qp3CEREo4YtG4goI3V2dqKrq0vvMIhUdXV18fzU6MiRIzh79qzeYRCNyNmzZ3HkyBG9wyAiGlVs2UBEGau4uJh3kSgllZSUAOBdTi0MBgN+8pOfoLS0VO9QiIatpaUFZWVleodBRDSq2LKBiIiIiIiIiBKKyQYiIiIiIiIiSigmG4iIiIiIiIgooZhsICIiIiIiIqKEYrKBiIiIiIiIiBKKyQYiIqI0Vltbi9raWr3DSBkGgyHspcbv96O+vn6UI6N0UF9fj2AwqDpPy7lFREQ3MdlAREREwxYMBlPyh5csy5BledB0v9+PTZs2IScnR/nRGC1ZE/njMhW3UwgGg+jq6kJjYyOMRqPqMv39/bBarTAYDLBarejo6FBdzuPxwGg0wmAwwGg0wu12Z0xMfr8ftbW1yvGMLGfBggWoqKiA3+8f9Nlo5xQREUUhExFloCVLlshLlizROwwiVZl0fra1tcnJrE4AkJubm+NaPlo8gUBAliRJ7uzsVN67XC4ZgGyz2VQ/4/P5ZACyz+eLP/hRZLPZZJvNFnX7A4GA3NbWpvxfbLeYJtjtdhmA3N3dLcuyLHd3d8sAZLvdnvYx+Xw+5djLsqysL7Kczs5OWZIkORAIqJYT6xyLprm5Oal/J0REKajFIMtM0RJR5ikpKQEAtLa26hwJ0WCZcn4Gg0FUVFTA4/Ek7Y6vwWBAc3MzSktLNS8PQDWe+vp6BAIB1NXVqX7G5XKhvLxctcx0qS5F236PxwNJkoZcNto0SZLQ1taW1jF1dXWhuLhYU2xWqxV33303NmzYoHl7YmlpaUFZWVnanEdERAnQyscoiIiI0pTf74fb7VaaqEe+93g8SrPz/v5+ZRnRJB0AGhsblSbsPT09Stlqjw5ETrPb7fB4PGHzgNTsR8Lv96OmpgZz585VnW+322EymTQ3zw8Gg3C73cp2NzY2hjW913IsQpetr69X5kd7lGAkIn/UCxaLJey93W4HcOOHOQAl1sgETTrGFJloEH0z2Gy2QcuWlJSgpqZG9XEKIiLSZpzeARAREdHwVFZWKj/2I993dXVBkiT09fVh2rRpmDp1Knbv3o2CggJl+a6uLlRVVaG0tBQ//elPMWPGDJw+fRrTp0+Hz+cLWxaAUpZQV1eHLVu2AIjvLq8e3nzzTQDA3XffrTp/w4YNCAQCMJlMmDlzJoqKimKWV1FRAUmSIMsy/H6/su+dTidyc3M1HQsAymeXLl0KWZbR0dGB+fPno7u7e8gYRkL80H788cfDpov9MGfOHHR2duJvf/sbfD4f8vPzkxaLHjH19/ejsbERwI1jGUmcJ2+++WbUpAgREcXGlg1ERERpKrIJeeh7cRe3sLAQANDQ0AAgPCkglsnNzVXuJosfyGo/5ERZQ6mrq0vKnfCReOuttwDE3oaamhpIkoRZs2aFtfKI1NHRAY/Hg0WLFgG4sa82btwIj8eDV199FYC2YxFalnh8Y968eQCAI0eOxL2N8Th58iQkScL3vve9QfPq6upgsVgwZ84cnDp1CrfccktSYxntmPr7+zFt2jQlURaasBNyc3MBIOZ5QEREsTHZQERERMpd9JqaGp0jSQ7xwzKW3NxcNDU1AUDMJvSir43QhMzMmTMBAIcPH44rLrF85CMqWuIdiW3btmHjxo3Kj+pQ9fX1ePTRRxEIBADcuPMfbTjIdIypsLAQsiyju7sbNpsNNTU1SisHQcSQqX8PRESjgckGIiIion/Lz89Hd3c3PB4PKisrVX/QhrZMEMSPU7W75LGI5eV/D6sY+koWt9sNSZIG9WEg5tXU1GDhwoXIzc1VOgBtaWlJWjx6xVRUVKQ8QmE2m0dUFhERDcZkAxERESkiO+fLRkVFRWhra4PH41E6JwwlnuFXa/kw3P03Ws31vV4vTp06haqqKtX5JpMJwM3kiei3I5k/xvWMafr06SMug4iI1DHZQERERMqP3cjO+TKFSBpobXovSRJcLpfq4wxLly4FAPT29irTRLliWFOtHA4HAMDpdCpliNEpEs3v9+P48eNh/Wl4vV5YrVblfWRniOIHfrI6SdQ7JrHPXS6X6ny1kSqIiEgbJhuIiIjSVORQi6HvxY+o0B/XkXfixTCPwWAQTqcTkiSF/YATd+lFIkIMPQhA+TEYepdf/EBOxaEvxR3syGSD2CdqrRTKy8tVf2wuXLgQkiRh69atyudeffVVWCwWpYNHrcdCdDK5ZcsW5OXlwWAwoKCgQElaiCExvV7vkNsYWr7adlZWVqKmpiasf4hZs2aFJZjWr18P4Oa5IY65mJ7OMRmNRtTX1ytDZwaDQdjtdthsNqWDTkEs8/DDDw+5jUREpI7JBiIiojQVOjRlQUFB2Pu8vLywfyOXB250amg0GpGXl4fCwkI4nc6w+T/72c8gSRJmzJgBj8eD4uJi5Y7/5s2bAUC5I71jxw7VIQRTxXe+8x0AwMDAgDJN/LAHbuwb0TljqLq6OtU7601NTZAkKexzv/jFL5RltB6L/Px89PX1KUkNi8WCvr4+ZeSKQCAAi8UyZPLGYDCElS8SF8KmTZui9icxY8YM5f/z5s1De3s7Tpw4AYPBgAMHDqC9vV1JoqRzTFVVVaipqcG0adNgMBjQ1NSEH/zgB6ojp4jzRJw3REQUP4Oc6gNjExENg7grKHqNJ0olep+f4gdfOlQBDAYDmpubUVpaqnl5QH3bRMuLDRs2JC7AUWI0GgcNdaq3TI6ptrYWeXl5qufKcP5+WlpaUFZWlhZ/c0RECdLKlg1ERESUFSorK3HixImwx0HSQVdXFzZu3Kh3GGEyOSav1wuv14vKysoEREVElL2YbCAiIsoikf08ZBPx+MPWrVs19TeQCjo6OjB58mTVISH1kskx9fT0oKGhAU1NTUpHlERENDzj9A6AiChViGd91Z7fJcoUkf08ZGqz7mhN3fPz8+F0OtHU1ISioiI9QotLaL8EqSKTY/J4PNi8eTPy8/MHzVPr04OIiKJjywYiohQRDAYTVpltbGyMu6zQ3uBDX3qI3BepFFu6k2U57JVptGxfbm5uWvbbQMm3YcMG1UQDkPl/O0REicaWDURE/6Z3i4bXX389IeV4vV6Yzea4PyfLMoLBoNJ7fCAQ0K0ZceS+kGUZfr9fuSuvZ2xERERENDS2bCAiSgHBYBCNjY0JKefIkSPD/nzoD3i9fsxH2xehdxuZaCAiIiJKbUw2EBHhRkd5brcbRqNR9b3H44HBYIDRaER/f7+yjMfjUZYRjy5YrVb09PQoZas1+4+cZrfblfHmR/KIQFNTE9auXas6r7a2NuYY9NGk474QCQvx+draWvj9ftTX14etTwyFCCBsXuh2ielGoxEdHR2DtjcYDMJqtQ5r3xIRERFlLJmIKAMtWbJEXrJkieblJUmSAcjiazH0fWdnpyzLstzX1ycDkC0WiyzLsjI/dJlAICBbLBYZgHz69GlZlmXZ5/OFlR1aVui0yPfxam9vV+JQK8tms8k2m23IciI/m0r7Qus+Euv1+XyDYu3s7Ax7H0qSJNnn8ymxSpIku1wuWZZv7F8Acnd396B90t3drVpeNPGen9kMgNzc3Kx3GEQj0tzcPKLvdyKiNNRikGX2cENEmaekpAQA0Nraqvkzkb3Xq/Vmr2UZr9eLWbNmwW63K53QDbcsrfx+P44dO4aqqqoRl6UlVrVpo7EvtG5XbW0tPvnkE+zevVv1c/X19aipqUFfXx8KCwuVWN977z2Ul5cDANxuN0wm06A4bTYb6urqlDKH03/EcM7PbGUwGNDc3IzS0lK9QyEatpaWFpSVlbFjSSLKJq1MNhBRRtIz2ZDosrRobGxUEg0jLSuRyQatyyU62SD09/ejtbUVNTU1YZ8TSRCHw6Hst/r6epSUlCjJB6PRqDzOEUmW5RHt45KSkhH1rUFE6YnVbiLKIq0cjYKIKM15PB489thjeoeRchobG+HxeGC325Vkg1BUVASLxQKz2azcMf/ggw+URAMAJdGQrB8HxcXF+MlPfpKUsjNJWVkZ1q9fjzlz5ugdCtGwdXZ2Ytu2bXqHQUQ0qphsICJKEovFMirrEZ0yqjEYDClxJ2209oXVasXu3bvhdrthNpvDHpNQi6mhoQGvvvoqcnJysGLFCtXlenp6MH369ITH+vWvf52PBmhQVlaGOXPmcF9R2mOygYiyDUejICJKMDH6wuOPPz4q65NledArdJ6eRnNfdHV14dFHHwUAmEwmAIiaaAButm4wmUxobGxEcXFx2HyHwwEAcDqdCAaDAG6OTkFEREREsTHZQESEGz8iQ/8f+l780BT/Ri4P3OhMUCzjdDohSRIkSVLmizv74sd3V1eXMs9qtQKAsnyyftBqGfoydBtDf2BHTtNjX0SuJ1RXVxfmzJmDmTNnhn2+v78/bOjNyDJEa4bQ+IRFixYBALZs2YK8vDwYDAYUFBSgpKQkZixERERExGQDEREAoKCgIOz/oe/z8vLC/o1cHgBmzpwJo9GIvLw8FBYWwul0hs3/2c9+BkmSMGPGDHg8HhQXF0OSJLhcLmzevBkAUFdXBwDYsWMHKioqEruBGhgMhrBtDP2BHTot9F9gdPZFZBwGgyHsJZ7nv+OOO8I+39jYiLy8PNhsNlgsFly6dCksFrFu0SIiVH5+Pvr6+mCz2QDcSJKIxzJCY4n1GAsRERFRtuJoFESUkUZraMGRjEiQadJxXwSDQfz0pz9VhsgcLRz6UjsOfUmZgENfElEWamXLBiIiylotLS3KD38iIiIiShwmG4iIhimyn4dslk77ora2Vnn8or+/H/PmzdM7JEqgyEds1LCjT4qmvr4+rE+aUFrOLSIiuonJBiKiYYrs5yHRIiu20V6pINn7IpHECBUOh0Pp2yHbBIPBpJ47yS5fi8iRWQS/349NmzYhJydH+RuK1nFqqv69qQkGg+jq6kJjY2PUfkT6+/thtVphMBhgtVrR0dGhupzH44HRaITBYIDRaFQ6fc2EmPx+f1jCMbKcBQsWoKKiQjVpGu2cIiKiKGQiogy0ZMkSecmSJXqHQaRK7/Ozra1NTmYVIJHlA5Cbm5vjWj7augOBgCxJktzZ2am8d7lcMgDZZrOpfsbn88kAZJ/PF3/wo8hms8k2my3q9gcCAbmtrU35v9huMU2w2+0yALm7u1uWZVnu7u6WAch2uz3tY/L5fMqxl2VZWV9kOZ2dnbIkSXIgEFAtJ9Y5Fk1zc3NS/+aIiFJQCzuIJKKMxA74KJXpeX4Gg0FUVFTA4/Ek5S5tosuPt4PIWB2V1tfXIxAIDGrRIj7jcrlQXl6uWma6VJeibb/H4xk0xKvastGmSZKEtra2tI6pq6sLxcXFmmKzWq24++67sWHDBs3bEws7iCSiLMQOIomIiNJFMBiE2+1WmoA3NjaGNfdWa+4fOc1ut8Pj8YTN8/v9SjN14MaQoaJZe09Pz4jLB270lRHtcYXR4Pf7UVNTg7lz56rOt9vtMJlMmpvnD3Us/H4/3G63sk89Ho/yCEB/f/+g2Orr65X50R4lGInIH/WCxWIJe2+32wHc+GEOQIk1GY8cjXZMkYkG0TeDGN42VElJCWpqalK+DxoiolTGZAMREVGaqKiowIULFyDLMnw+HzweDyorK5UfTT6fb9Bn+vr6wt6H/kCT//0MekFBAYxGIzweD7q6ulBVVYVAIAAAmDFjhpJwGG75qeDNN98EANx9992q8zds2ACbzQaTyQSv1ztkeUMdi8rKSphMJmWfSpKEvr4+eDwe/PznP1fK8fv9qKysxNSpUyHLMtavX4/58+drimEkRJyPP/542HSxH+bMmYOuri688cYb8Pl8KCoqSmo8ox1Tf3+/ksSoqKgYNF+cJ+K8ISKi+DHZQERElAY6Ojrg8XiwaNEiAEB+fj42btwIj8eDV199VZkWSXSIGUtoQkDc/c3NzVXuMIuWCsMtH7iRhNCzQ8633noLQOx4a2pqIEkSZs2aFdaiI5KWYxHavF/sU7HuhoaGQWWJxzfE6ChHjhyJexvjcfLkSUiShO9973uD5tXV1cFisWDOnDk4deoUbrnllqTGMtox9ff3Y9q0adiyZQuAm+d3qNzcXACIeR4QEVFsTDYQERGlAdG/Q+gP/pkzZwIADh8+nJR1ijvHNTU1SSl/NIkflrHk5uaiqakJAGI2oU/ksRDLRz6OoiXekdi2bRs2btyo/KgOVV9fj0cffVRp3VJRURF1OMh0jKmwsBCyLKO7uxs2mw01NTVobGwMW0bEkAnnPhGRXthBJBFlJHYQSalsOOdntE7pIqdr6WBPayd8iSx/uBLVQWSsmCI7gPR6vZg1axYkSYLT6UReXl5a7Cut5bndbly4cAFVVVWq80wmEwKBAHJzc9HT04MZM2bA4XCoLp+uMQmiLLX4hnMuRcMOIokoC7GDSCIionQgOtNTu9se2aFeoiW7/FRTVFSEtrY2eDwe5bn+UMk4FqPVXN/r9eLUqVNRf6SbTCYAN+/sFxQUAADMZnNGxjR9+vQRl0FEROqYbCAiIkoDS5cuBQD09vYq00QzctFSItHED+DIDvvSkUgaaG16L0kSXC6X6uMMiTwWDocDAOB0OpUyxOgUieb3+3H8+PGwvjO8Xi+sVqvyPnKECPEDP9rIEekek9jnLpdLdb7aSBVERKQNkw1ERERpYOHChZAkCVu3blXuqL/66quwWCxKp4LAzTvrIlEghgsEoPyAC70zH/mjVgz9GAwG4XQ6IUlS2I+64Zav99CX4g52ZLJB7Eu1Vgrl5eWqPza1HIvQ8sQ6Q9ct5otOJrds2YK8vDwYDAYUFBQoSQsxJKaW0SlCy1fbzsrKStTU1IT1DzFr1qywZNL69esB3DwPxPEV09M5JqPRiPr6emXozGAwCLvdDpvNpnTQKYhlHn744SG3kYiI1DHZQERElAZE54WSJKGgoEB5bvwXv/hF2HI/+9nPIEkSZsyYAY/Hg+LiYuUu/ebNmwHcHJ5yx44dg4b9mzlzJoxGI/Ly8lBYWAin05nQ8vXyne98BwAwMDCgTBM/7AGE7dNQdXV1qnfWhzoWolwAyMvLC/s3dH5+fj76+vqUpIbFYkFfX58yckUgEIDFYhkyUWMwGMLKF4kLYdOmTaqjLgBQ+iwAboyG0d7ejhMnTsBgMODAgQNob28PS2ila0xVVVWoqanBtGnTYDAY0NTUhB/84Aeqo6SI80ScN0REFD92EElEGYkdRFIqS8XzM9EdFSZKojqIBKC0stiwYUPiAhwlRqMxbDjNVJDJMdXW1iIvL0/1XGEHkUREmrCDSCIiIsoOlZWVOHHiRNijH+mgq6sLGzdu1DuMMJkck9frhdfrRWVlZQKiIiLKXkw2EBERZbnQ/gXU+i7IFOLxh61bt2rqbyAVdHR0YPLkySguLtY7FEUmx9TT04OGhgY0NTUpHVESEdHwjNM7ACIiItJXaP8CBQUFGdHUO1pT9/z8fDidTjQ1NaGoqEiP0OIS2i9BqsjkmDweDzZv3oz8/PxB89T69CAiouiYbCAiIspymZBcELRsS25ublr220DJF+u8yKS/EyKi0cDHKIiIiIiIiIgooZhsICIiIiIiIqKEYrKBiIiIiIiIiBKKyQYiIiIiIiIiSih2EElEGevs2bNoaWnROwxKYVevXsXly5eRk5Mzqus9e/YsAPD81Kizs1PvEGIKBoMcJpFiSvVzmIgoGQwyu9YlogxUUlKCI0eO6B0GERGRgtVuIsoirUw2EBFRVrh+/To6OjrgcDhw9OhR5OfnY/ny5bBarZg2bZre4VEaO3nyJBwOBw4dOoRr165BkiRUV1fjkUce0Ts0IiIivTDZQEREme2jjz7CCy+8gN27d+Pvf/875s2bB7PZjCeeeALjxvFpQkqc8+fPw+1247nnnsM777yD2bNnw2w2Y+nSpZg0aZLe4REREY0mJhuIiCjzhLZieOmllzBp0iRUVFSguroad911l97hURYQrR0OHjyIcePGoby8HKtXr0ZRUZHeoREREY0GJhuIiChzBAIBHDhwADt27MCHH36o3FmuqKjAhAkT9A6PslAgEEBLSwueffZZnDp1Sjknly1bhokTJ+odHhERUbIw2UBEROlP3EV2Op0YP348ysvLsWbNGjz44IN6h0akOHnyJLZv3w63242JEyeirKwMa9euxf333693aERERInGZAMREaWnaM/HP/nkk6M+lCVRPHw+H/bv3489e/bgzJkzyrm7fPly3HrrrXqHR0RElAhMNhARUXqJ7Pm/pKQE69evx0MPPaR3aERxCe1b5MUXX8Rtt92GlStXwmw2s28RIiJKd0w2EBFR6rt06RI8Hg+2b9+O3//+97jnnnuwcuVKVFVVYfLkyXqHRzRi0UZNWbx4McaPH693eERERPFisoGIiFLX6dOn8fzzz6OxsRH//Oc/YTQaYTabMX/+fBgMBr3DI0q40NYOR48eRX5+PpYvXw6r1Ypp06bpHR4REZFWTDYQEVFquXLlCo4dOwaHw4H29nZ885vfRGVlJZ566il85Stf0Ts8olFz7tw5HDx4EDt37sTAwIDS2uGJJ57AuHHj9A6PiIgoFiYbiIgoNXzwwQdoamrCvn378Omnn2Lu3LlYt24dfvjDH7IVA2W1yATclClTsGzZMqxZswbf+MY39A6PiIhIDZMNRESkn8gm4wUFBaioqMDq1atRWFiod3hEKef999/H3r17sXfvXnz22WdYuHAhqqur+WgRERGlGiYbiIho9A0MDMDpdGLXrl04e/Ysm4cTxeny5ctoa2vj40ZERJSqmGwgIqLREW2Yv6effhp33nmn3uERpa2//vWv2L9/PztSJSKiVMJkAxERJZfP58P+/fvhcDjQ29uL2bNnw2w2Y/ny5bj11lv1Do8oY4ghYh0OB44fP44ZM2bgxz/+MSorK3HbbbfpHR4REWUXJhuIiCg5Tp48CYfDAafTiS996UsoKyvD2rVrcf/99+sdGlHGE39/hw8fxtWrVyFJEsxmMxYsWKB3aERElB2YbCAiosQJBoNobm7Gzp078ec//1lpxfDkk08iJydH7/CIss6FCxfgcrmwe/dudHd3495778Xy5cthNpvx5S9/We/wiIgoczHZQEREIyfuoh46dAhjxoyByWSCxWLBt7/9bb1DI6J/498pERGNIiYbiIhoeC5duoSWlhZs27YNb7/9NmbOnIkVK1bwjilRimMLJCIiGgVMNhARUXxEz/cOhwMXL15Uer7ns+BE6Uetb5U1a9bgwQcf1Ds0IiJKb0w2EBHR0C5fvoy2tjall/tvfetbWLVqFVatWoXbb79d7/CIaIQ+++wzvPDCC9ixYwc+/PBDpbVDRUUFJkyYoHd4RESUfphsICKi6N5//33s3bsXe/fuxWeffYaFCxeiuroa8+fPh8Fg0Ds8Ikqw69evo6OjAw6HAy+99BJycnJQWlqKdevW4b777tM7PCIiSh9MNhARUbhr167hlVdewbPPPov29nZMmTIFy5Ytw5o1a/CNb3xD7/CIaJR8/PHHOHDgABoaGvC3v/0NjzzyCKqrq7F48WKMHz9e7/CIiCi1MdlAREQ3DAwMwOl04rnnnsO5c+cwb948mM1mPPHEExg3bpze4RGRTkJbO7z44ou4/fbbsWLFCjz99NO488479Q6PiIhSE5MNRETZLNqPCIvFgjvuuEPv8IgoxYik5K5du3D27FkmJYmIKBomG4iIspHP58P+/fuxZ88enDlzhs2jiSgu165dw29/+1s4HA4cPXoUBQUFqKiowOrVq1FYWKh3eEREpD8mG4iIssnJkyexfft2uN1udvxGRAnxwQcfoKmpCc8//zw++eQTpbXDj370I4wdO1bv8IiISB9MNhARZbpgMIjm5mbs2LED7777rjKk3bJlyzBx4kS9wyOiDHHlyhUcO3YMDocD7e3tmDp1KlatWoXVq1cjPz9f7/CIiGh0MdlARJSpTp48CYfDgYMHD2Ls2LEwmUywWq2YNWuW3qERUYbr6enBvn37sHfvXpw/fx6LFi2C2WzmsLlERNmDyQYiokxy4cIFuFwu7N69G93d3bj33nthNpuxatUqTJo0Se/wiCjLXL58GW1tbXA4HDh+/Di+9a1vYdWqVVi1ahVuv/12vcMjIqLkYbKBiCgTvPfee2hoaMC+ffvwxRdfwGg0wmw2Y8GCBXqHRkQE4Mb31IEDB+BwOHDx4kV+TxERZTYmG4iI0lXkHcPp06fjqaeeQmVlJW677Ta9wyMiUnXp0iW0tLRg+/bt+NOf/oR77rkHK1euRFVVFSZPnqx3eERElBhMNhARpRs+C01EmUL0LXPo0CFcu3YNkiShuroajzzyiN6hERHRyDDZQESUDiJ7eZ8yZQoqKyvZyzsRZYTz58/D7XZj165d8Hq9yqg5S5cuZX8zRETpickGIqJUdu7cORw8eBA7d+7EwMAAx68noowXOpLOuHHjUF5ezpF0iIjSD5MNRESp5vr16+jo6IDD4cDRo0eRn5+P5cuXw2q1Ytq0aXqHR0Q0KgKBAFpaWrBjxw68++67SmuHZcuWYeLEiXqHR0REsTHZQESUKj7++GMcOHAADQ0N6Ovrw/z582E2m7F48WKMHz9e7/CIiHRz8uRJbN++Hc3NzZgwYQLKyvI4u/4AACAASURBVMrwzDPP4IEHHtA7NCIiUsdkAxGRnmRZRnt7OxwOB1566SXk5OSgtLQU1dXVuPfee/UOj4gopfh8Puzfvx8OhwO9vb1Ka4eKigpMmDBB7/CIiOim1jF6R0BElAlOnToFp9OpeflAIACHw4H7778f3//+99Hb26v0y7Bnzx4mGoiIVBQUFOB//ud/8P777+O1117DXXfdhWeeeQZTp05FdXU1PvzwQ81l/eUvf4HL5UpitERE2Y0tG4iIRuj//u//IEkSJk+ejL/97W8xh59U6/hszZo1ePDBB0cxYiKizPHRRx/hhRdewO7du/H3v/9d6Uh3qEfQ1q1bh507d+IXv/gF/vu//3sUIyYiygp8jIKIaCTcbjeWL1+Oa9eu4fr16/jf//1f/Nd//VfYMhzSjYgo+SI71/3KV76CFStWwGKx4I477ghb9uLFiygoKMDnn38Og8EAi8WCHTt2cJQfIqLE4WMURETDtX37dixduhRXr17F9evXMW7cOOzatUuZf/LkSTz99NNK897p06fjd7/7Hf74xz/CbDYz0UBElEBjxozBggUL0NLSgr6+Pqxfvx6HDx/GN7/5TXz/+99Ha2srrl69CuBGovjixYsAbvSd43A4sGjRImUaERGNHFs2EBHF6dq1a1i/fj127tw5aN6YMWPwy1/+Es3NzfjDH/6A++67DxaLBRUVFcjNzdUhWiKi7HXlyhUcO3YMDQ0N+O1vf4vCwkJUVVWhpaUF7777Lq5fv64sO27cODzwwAP4zW9+g/z8fB2jJiLKCHyMgogoHpcvX8ayZctw9OjRsEqqMH78eEyaNAkLFiyA2WzG/PnzY/bhQEREo+P999/H3r174XA48Nlnn6kuM378eHz1q1/Fa6+9hhkzZoxyhEREGYXJBiIirT799FP88Ic/xB/+8AelKa6a/Px8DAwM8NlfIqIUZDabceDAAVy5ckV1/rhx4zBx4kS8/PLL+H//7/+NcnRERBmDfTYQEWlx5swZPPzww/jjH/8YM9EAAP/4xz/w8ssvj1JkRESk1eeff45Dhw5FTTQAwNWrV/H5559j3rx5aGlpGcXoiIgyC5MNRERD+MMf/oD//M//RH9/P7744oshlx8zZoxqfw5ERKSvgwcP4vLly0Mud/36dVy9ehUmk4nf50REw8THKIiIYnj55ZexZMkSXLlyRbWPhmgMBgM+/PBD3HnnnUmMjoiI4nH//ffj1KlTcX+upqYGv/zlL9kHDxGRdoP7bGhpaUFZWZleAREREVESJeseA+sPRERE2UulftE6LtrCzc3NyY2GiNLKr3/9awDAT37yE50jGR2ff/453nnnHdxyyy249dZbMWHCBEyYMAG33norbrnlFkycODHqZ8vKyrB+/XrMmTNnFCMmiq2zsxPbtm1L+npYf6B0d+nSJVy+fBmXLl3CxYsXcfnyZVy+fBn/+te/MGXKFBQWFuodYkbJtvrFSLB+QakoVv0iarKhtLQ0aQERUfppbW0FwO8GLcrKyjBnzhzuK0o5o5Fs4HlPRPFg/UI71i8oVUWrX7CDSCIiIiIiIiJKKCYbiIiIiIiIiCihmGwgIiIiIiIiooRisoGIiIiIiIiIEorJBiIiIiIiIiJKKCYbiGjU1dbWora2Vu8wUpLf70d9fb3eYVAKqq+vRzAY1DsMIqKUxfpFdKxfUDTJrF8w2UBEWScYDMJgMOgdxiB+vx+bNm1CTk4ODAYDDAZD1EqTmB/6SlXBYBBdXV1obGyE0WhUXaa/vx9WqxUGgwFWqxUdHR2qy3k8HhiNRhgMBhiNRrjd7oyJye/3o7a2VjmekeUsWLAAFRUV8Pv9wyqfiIiSi/WL0ZWK1/JUjEnX+oUcobm5WVaZTERZbsmSJfKSJUv0DiMh2trakvo9B0Bubm6O6zOBQECWJEnu7OxU3rtcLhmAbLPZVD/j8/lkALLP5xtxzMlks9lkm80mA1Dd74FAQG5ra1P+L7ZbTBPsdrsMQO7u7pZlWZa7u7tlALLdbk/7mHw+n3LsZVlW1hdZTmdnpyxJkhwIBOIqX5aTf31n/YGIhoP1C+1YvwiXatfyVIxJ5/pFC5MNRKRJplQGxEU31SoDdrtd9aIvLlYulyvqutJFtAtv5AU22rLRpkmSlPYxhVYEhorNYrEMqwLEZAMRpSLWL7Rj/UJdqlzLUzEmnesXLXyMgohGld/vh9vtVpqWRb73eDxKc7H+/n5lGdGUDAAaGxuVpmc9PT1K2WpN/iKn2e12eDyesHmAvs95+v1+1NTUYO7cuarz7XY7TCaT5uZzwWAQbrdb2b7GxsawpnFa9nnosvX19cr8aE39RkKSJNXpFosl7L3dbgcAdHV1AYASa11dXdrHVFxcHPZePDtps9kGLVtSUoKamho+TkFEFIL1i8FYv2D9Qvf6RRyZCSLKYom68yCy/uJ7JvS9yL729fXJAGSLxSLL8s0MbOgygUBAtlgsMgD59OnTsizfbPYX+h0mygqdFvlelm82e0sExHnnQTS77OvrUy1LxIeQ5nSR80NJkiQ7HA5Zlm/sE0mSwprGadnnoZ8Vdz3a29tVY9BKbb+rCQQCqk0KZfnmfujs7JRdLteIm3imYkx9fX1KmeLcjpwfLZZY2LKBiFIR6xfasX6hLhWv5akYkw71Cz5GQUTaJLKZo5aLs5Zl1J5fG25ZiRRvZUB88UcrS5bDm2eGXiAiPycu2KEXpM7OzkFNJbXsJ/FcX+Qyw600ad3v7e3tMZ8bFJVAm802rGcLUzmm0Mpr5LktiIpJvE0dmWwgolTE+oV2rF+oS7VreSrGpFP9go9REFH6KioqAgDU1NToHMnIbNmyZchlcnNz0dTUBAAxm7i1trYCAPLz85VpM2fOBAAcPnw4rrjE8pFNRbXEOxLbtm3Dxo0bkZubO2hefX09Hn30UQQCAQBARUXFqAwHOVoxFRYWQpZldHd3w2azoaamBo2NjWHLiBjS/bwnIkpVrF8MxvpFesekW/0ijswEEWWxVLzzkOiyEgVx3nmIFU/kdHG3RWTBh9rWaNP12E9aynO5XEoTTbV5AJTM/unTp2UAUZdP15gEUZaW46kFWzYQUSpi/UI71i/UpeK1PBVjEkaxfsGWDUSU/iI71clkRUVFaGtrg8fjUToPCiU6HlK7MzHc/RTaSVYyeb1enDp1ClVVVarzTSYTgJuZ94KCAgCA2WzOyJimT58+4jKIiGj4WL+4ifWLzIlpNOsXTDYQUdoSF6nHH39c50hGRlzUtTaNkyQJLpdLtbnh0qVLAQC9vb3KNFFuSUlJXHE5HA4AgNPpVMoQvUcnmt/vx/Hjx8N6WfZ6vbBarcr7yB6cxQU4Ws/O6R6T2Ocul0t1vlpP0kRENHKsX7B+oTY9U2Ia1fpFHM0giCiLJaqZY2iPzj6fL+y9aCommu+JZWR58HjQgUBAttlsg8YbjuxBWnReBNzsCVl0hOTz+ZROcFKxt2ixb6L1PqzW8ZPo6EmSJOVzLpdrUC/QWvZ56HKhLxGn3W6XAW29R4eWH9nBkeiVWm1doT0ii86pxDkgjm17e7uyTLrGJEmSbLfblX0rzm+1c5KjURBRJmH9QjvWLwZLpWt5Ksakc/2Co1EQkTaJqgyofbmGvtSWCZ3W3d2tfEk7HI5BX+J9fX3KfPFlKYZXEhc58VyizWZTpulZGRAXXTFMlChDbT9EiqwMifIcDkdYBSp0P2nd57IcPkySxWIJq7DYbDbZYrGoxhAq1rGW5ZsVOLVX5NBM7e3tyvIWiyXsopvOMYkKoXjZ7faw8yGUqHDEOwQWkw1ElIpYv4hvG1m/uCnVruWpGJPO9YsWgyzLMkK0tLSgrKwMEZOJKMuJJnKiN+LRJnoqTofvJoPBgObmZpSWlmr+jGg6uGHDhmSFlTRGoxFtbW16hxEmk2Oqra1FXl5e3OdKsq/vrD8Q0XCwfqEd6xf6y+SYklC/aGWfDUREKaCyshInTpxAV1eX3qHEpaurCxs3btQ7jDCZHJPX64XX60VlZWUCoiIiokzH+kXiZHJMyapfMNlAcamtrUVtbW3S1+P3++F2u2E0GpO+Lr2M1r6MlI77NrTn42jjP6c7Mc711q1b4fV69Q5Hk46ODkyePBnFxcV6h6LI5Jh6enrQ0NCApqYm1fG4M0U6fkfRyLF+kTisX2jH+kVqyuRreSKlQ/0i45MNwWBQaR41HF6vFwaDQXmF9hIqeDweGI1GGI1GeDweZXp/f3/YZ8Urktr8/v5+WK1WZZ0dHR3D3oaR7gM9bNq0CSaTKWx/DkVtX2vZ/6MhlY7BcPat3sRwP5H/zzT5+flwOp04fvy43qFoMm/evJQbnjGTY/J4PNi8eTPy8/MTEFXqSsfvKK1Gci0IBoPo6upCY2Nj1B9zWuoOWsqJJ6ZUubZpxfpF8qTj3y7rF6kpk6/liZQW9Ys4OnhIS6JTjOEK7QQFGNw7p8vlkiVJkgOBgBwIBGSLxSI7HI6wZURnG5HTQ7W3t4f1givWEwgEZJfLpbpurUa6D/QCRO+0JprQ3l8jiV5d9ZBqx2A4+zZRHThlA8TZgRPRaEinDiKH8x2VDkZyLRCdzEXbN1rrDkOVE49Uu7ZpxfpF8rB+kVysX1AqitVBZEa3bAgGg2hsbBxRGV/96lchy7LyCh3btL+/HyaTCRs3bkRubi5yc3NhsVhgNpvDmikVFxfDbrfjT3/6U9T1vP3225g3bx4A4PXXX1fWk5ubi/LycgAY1h2IROyDdBKr6Y/Yv6Mt244BERENNtJrQV1dXdiY7JG01h2GKkerbLu2sX5BRBS/EScb/H6/8hhBMBiE1WoNe07M7/ejvr4eBoMBRqNxUJM+Ma+xsRF+v19pChb53JfH41HK6O/vHxSD2jrsdrvSlGs4Tdz6+/thNBpRW1ur2qnKG2+8AQCYMmWKMu1rX/saAOCtt94KW7akpAQNDQ1wu92DygkGgwgEAkrTldCERiiLxRJX/ID6Poh1zMSFSyxbW1urPMMWeUwScYxC94Hb7Vbm9/T0DNqWkTyDGNnTsFqzx8hp8WxfaPzifBaiHQO15xrVyol8nlBLTLGOIxFRKtHy/Q9Ev47E810drc6RyHUkq04ylETWHbRg/QLKtgOsXxARqYqjGYQqMd4s/j2Ga3d3t2yxWGRZvjEWqxh/VpZvNjPr7u6WZVmW7Xa7MqZqIBBQmvWplSvLN8Zjxb/HGRWGWgdG0EwwclxSSZLCxh0V455GEstGstvtMoBB4/a6XC4lXjWi6d5wH6OI3AexjpnYJp/PN2h/h35OrRxZHt4xEmVZLBZl34jmn6Fxax2nOPJzIqZQYtxhteXi3T6xbGhsFosl7H2sYxBZjnjcRuw38ZhOPDHFOo5q8WjBZo7agc0cKQWl6mMUWr7/Y11HtH4vxqpzJGodyayTCFrLGKrukIxYWL9g/YL1i+Ri/YJSUazHKBLSZ4P4YlH7ER1ZFgDlS1J8WQniCzqyXLV1xbOOkVxMA4GA3N3drVRKQvtdiFZ2tOmnT59WvfCrJSZCtbe3h10Q4hVrP0aWabPZYl40hnqvNm2oYySSOqdPn1bmx3o2cijic5GvaMvFmhbP9oWey52dnWHHVUs5opIUWQ4ApSKltax4j6MWrAxox8oApaJUTDZo/f4fzrVe7XsvVp0jEetIdp0knjKGqjskKxbWL1i/YP0ieVi/oFQUK9kwDgkU+Tzb4cOHAWBQU8EtW7agrq4OFosFBQUFcLlcWLhwIfLz85VmaFoNtY6Rys3NRVFREYqKilBYWAiPx4OqqqphlTV9+nRYLBY0NjYqzR27urqwdOnSmJ/btm2b0i9EokWWKfZZf38/WltbE7KOoY7RK6+8AgBhvakmYlvFudTf349p06aNuLxoxPaF9uBaXFyMtra2uMoR+zu0nJkzZyrrEM/fapGM4wgAZ8+eRUtLS8LKy2SdnZ16h0AUJhXPSa3f/4m41g9V50jEOpJdJ4lHMusOWrB+MXKsX5CaVPwup+wW85yMIzMRFeLI7IY6ffp0WLMtu90+5Ocjpw21jqHmx0NkwwURu9o6I5vBCd3d3TIAub29XZblG83RRLNONS6XK+YoFlpo2Y+hHA6HLEmS0hIj1v5O5jEa7rGLFpPW5RK9fSMtZzgxyXJ8x1GLJUuWRL2rwxdffKXPK1mSWX8YKna1+ZHThlPniHcdwykjXlrK0FJ3SFYsscpl/YL1CzWsX/DFV2a8VCT2MYpo00Obr6kRz/QB4Rd/tXIjpw21jhgbPyyhSQQxLGZoszTx/Fqsi7zFYpElSQp7llGNeHxjpLTsR0E02RMJkGj7O56yh3uMhnvstH4unthjLSMqr7H63YinnNDzSSw31POQkdPiPY5asJmjdgCbOVLqScXHKLR+/w/nOhKt7KHqHCNZx2jUSYYqQ2vdIVmxsH7B+gXrF8nD+gWlIt2GvnQ4HAAAp9OJYDAI4GavwcCNZm/BYBBFRUXYvXs3uru7UVNTk9B1JFIwGERJSYny/rHHHgMA9Pb2KtMGBgbC5qmxWCzweDyora0NKy+U3+/H8ePHw5pder1eWK3WEW3DUEwmEwCgsLAwYWUOdYzE/NDhQtOJeCSmoaFB2b7+/v64j5V4nCb0fBLlRTtPoknGcSQiSjSt3/+JuNYPVedIxDpGs06iRq+6gxasX8SP9QsiSntxZCZUqfW6qzYv9BWaDbXZbMr7vr4+5S5D6GdFJ0OhnfqI7OxQ6wjN5kY2mYzF5XIpjzqI2NR6dHY4HEovx4FAQLZYLJoeexB3VdQ6bhK9BKtt13BGpIjcB7GOmVi2r68vrHmcz+cL+1zk+5EcI9EaRJIkZZrozAghWXctvUWHrn+oDjXFMRB3RERnSWKd8Wxf5PGyWCxhd1piHQNRTiAQUHqHFtNcLtegnre1xBTPcdSKdx60A+88UApKxZYNWr//Y11HtH4vxqpzJGodyaqTCLGucfHUHeK5VsbC+oU61i9Yv0gW1i8oFSV1NIrQL0C1URX6+vqUkRwi+ycI/XIE1Jszhl601KYNtQ7RR4LNZovriy902EubzRazCZtYVpKksARFLN3d3VGTEuIipfYa6pGUaOsK3QexjlnksqLX4dBhm6K9ZHl4x0jMF9stLsRiOCtx3IaqDMSKS01fX59y0RQVsdB1xrN9Yl+JfRd5nGIdg8hyxOM5wI1eokMrNVpjiuc4asXKgHasDFAqSsVkgyxr+/4Xy6ldR7R+L8aqcyRqHbHKkOXh10nU1he5Xq11h3ivlbGwfqGO9QvWL5KF9QtKRbGSDQZZlmWEaGlpQVlZGSImE1GWE00tE9n7dKYyGAxobm5GaWmp3qEQKZJ9fWf9gYiGg/UL7Vi/oFQU4/rfmtQ+G4iIiIiIiIgo+zDZQEREREREREQJlXXJBoPBoOmV6jJlO4hodI1mz/jZqL6+XunlnWgoqXgtT8WYiCj1sX6RXOlav8i6ZIMsy5peqS5TtoNIq2AwmNQKbrLLTwV+vx+bNm1CTk6O8oOhtrZWddl0+nEhhoIzGAywWq3o6OhQXc7j8cBoNMJoNMLj8QxZbmNjo+p2i3IMBgOMRiPcbrcyb8GCBaioqIDf7x/+BlHWSMVreSrGRJRMrF+MHOsXrF9Ek3XJBiJKT6+//npal6+3YDCIyspKrFixAhaLBYFAAC6XC1u2bFGtEMiyDJ/PBwDw+Xwp++MiGAzC6/Vi9+7dCAQCePTRRzF//vxBF3u3243GxkY4nU44nU688soraGxsjFqu1+uF2WweNL2+vh5GoxF1dXWQZRl1dXUwmUzK3ZyioiJs3LgRlZWVaXkHgogo27B+MTKsX7B+EQuTDUSU8oLBYMwv7lQvPxU0NTWhqKgIxcXFAIDc3FyUl5cDALZs2RKWPRfy8/PD/k1Fr7/+OiRJAhC+TUajUVmmv78fJpMJGzduRG5uLnJzc2GxWGA2m+H1egeVGQwGceTIEdX11dTUALhx0Q/998SJE8oyxcXFmDp1KpqamhKwhURElCysX4wc6xesX8TCZAMRJVUwGITb7VaayjU2NoY1AVNrRhc5zW63K5lkMd3v9yvNzYCbTdKsVit6enpGXD4A1NbWRm0GmE78fj9qamowd+5c1fl2ux0mk0m1QqBmqGPq9/vhdruVY+PxeJQmgf39/YNiq6+vV+ZHa6IYjagIRLJYLMr/33jjDQDAlClTlGlf+9rXAABvvfXWoM82NTVh7dq1quXa7XYAQFdXFwAo21NXVxe2XElJCWpqatKuuSMRUbpg/UJ/rF+wfjEUJhuIKKkqKipw4cIFpdmcx+MJawImmtKF6uvrC3sf+kUrnhcuKChQno3r6upCVVUVAoEAAGDGjBlKhWC45WeSN998EwBw9913q87fsGEDbDYbTCaTaiY+0lDHtLKyEiaTSTk2kiShr68PHo8HP//5z5Vy/H4/KisrMXXqVMiyjPXr12P+/PmaYohGxPD4448r08RdgcLCQmWauJsS2Ryyo6MDjzzySNS7LWJfzZkzB11dXXjjjTfg8/mUOxCC2Ndi3xMRUWKxfqE/1i9YvxiSHKG5uVlWmUxEWW7JkiXykiVL4vpMe3u7DED2+XzKtM7OThmA7HK5lGkABn3vRE7Tsowsy3J3d7cMQLbb7SMuf7gAyM3NzQkpKxFsNlvUbRPTA4GALEmSDEA+ffr0oPlCIo+py+VSXcZms8W5heHxSZIkBwKBmLGoTff5fLLD4Rjyc7IsyxaLRYk1dF1CIBAYdB7qLdnXd9YfiGg4WL/QjvUL1i/SrH7RwpYNRJQ0ra2tAMKfyZs5cyYA4PDhw0lZp8gAi2ff6MYzk0PJzc1VngGM1TwvkcdULB/Z7FRLvNFs27ZNeXYyXseOHUNVVdWQy9XX1+PRRx9V7nRVVFQM6qxJrJ/nIRFR4rF+kRpYv9Amm+sXTDYQUdI0NDQMmia+JLUMDUSjKz8/H93d3YOaLYZK5DEVy8sJGlbP7XZDkiSlkyoh2nOXwM1nLz0eDx577DFN66ipqcHChQuRm5uLiooKeDwetLS0DCtmIiKKH+sX6YX1i+ytXzDZQERJI76E1bLYoR3sJEOyy89URUVFaGtrg8fjUTorCpWMYxra4dZweb1enDp1SvXOgVrMouOlhx56CMCN3qWnTZsWtcMvwWQyAbhZASooKAAA1WGsiIgoOVi/SD+sX2Rn/YLJBiJKmqVLlwIAent7lWkim11SUpKUdYoLS2gHPtlOXNS1jsssSZIyRnakRB5Th8MBAHA6nUoZovfoePj9fhw/fjysIy6v1wur1QoAyh2F0JgHBgbC5sW6+xH6/8i7GKJSEO3uhs1mi2tbiIhoaKxfpAbWL1i/GAqTDUSUNAsXLoQkSdi6dauS9X311VdhsVgwb948ZTmRsRYXcjHsDwDlCz00exx5sRBDKgWDQTidTkiSFPblPNzyM2VoqunTpwMYXBkQx0TtLkJ5ebnqhUzLMQ0tT6wzdN1i/qJFiwDceIYyLy8PBoMBBQUFSqVCDFkVq/do0eN0TU1N2F2DWbNmKRXCwsJCOBwOHDhwAMFgEMFgEAcOHIDD4QjrQVqL9evXA7h5zolzSUwXxJ2Nhx9+OK7yiYhoaKxfpAbWL1i/GFIcvUkSURYbTm/RsnyzB178u+ddl8s1qHfdvr4+pafitrY2WZZlWZIk2eVyKb0Si16gbTabMk2U2d3drXze4XAkrHybzTasnouRYr1F+3w+GYDc2dmpTBP7LvSlRpIk1fJiHVO1cqOtq6+vT+nN2mKxyH19fco8m80mWywW1RgE0Wuz2iu012tZluW2tjYZgCxJktze3j7EXoveW3R7e7uyXovFolqW6EE7tFdtvXE0CiJKRaxfaMf6BesXaVa/aDHIcnhPGS0tLSgrK8u4cWCJaGRENlj0FpwKxLNuqfZ9ZTAY0NzcjNLSUr1DUYi7KRs2bNA5kvgZjUa0tbXpHUZcamtrkZeXl1L7O9nXd9YfiGg4WL/QjvWLxGL9IjFiXP9b+RgFEVEWqKysxIkTJ8KaeKaDrq4ubNy4Ue8w4uL1euH1elFZWal3KEREREnF+sXoScf6BZMNRJSWQp/bizZmM90kxrneunVrzGcUU0lHRwcmT548aKipVNbT04OGhgY0NTUNayxuIiLSF+sX8WH9YnSka/2CyQYiSktiSKDI/1N0+fn5cDqdOH78uN6haDJv3jyl86l04fF4sHnzZuTn5+sdChERDQPrF/Fj/SL50rV+MU7vAIiIhiPVnqNMF7m5uSn1nF+m4b4lIkpvrF8MD+sXyZWu+5YtG4iIiIiIiIgooZhsICIiIiIiIqKEYrKBiIiIiIiIiBKKyQYiIiIiIiIiSqioHUSWlJSMZhxElOLE+MnZ9t3w2Wef4cqVK3H3SP3rX/8ara2tSYqKKH5nz54dlfVk23cEZb4LFy4gGAzi61//ut6hZKRsrV8MF+sXlGpi1S8MckSXq52dnfjVr36V9KCIiNLBu+++i7/+9a+YNGkS7rzzTtxxxx245ZZb9A6LaNiSVUll/YEyyfXr13Hu3Dn09vbiH//4B3Jzc7FgwQIYDAa9QyMiSkkq9YvWQckGIiIKd/r0aTz//PNoamrChQsXsGjRIpjNZsyfP58VTyKiDHL27FkcOnQIO3fuxMDAAObNmwez2YwnnngC48ZxxHgiojgw2UBEpNWlS5fg8XjgcDhw/PhxzJgxAz/+8Y9RVVWFyZMn6x0eERENw/Xr19HR0QGHw4GjR48iPz8fy5cvh9VqxbRp0/QOj4goXTHZQEQ0HCdPnoTD4cDhw4dx9epVSJKE6upqPPLII3qHRkREGgwMDMDpdGL37t34+9//rrRiWLx4McaPH693eERE6Y7JBiKikTh//jzcbjd27doFr9eL2bNnw2w2Y+nSpZg0aZLe4RERUYjQVgwvvvgibrvtRc9Y8wAAIABJREFUNqxcuRJmsxl33XWX3uEREWUSJhuIiBJFtHY4ePAgxo0bh/LycqxZswYPPvig3qEREWU1n8+H/fv3Y8+ePThz5oySGF6+fDluvfVWvcMjIspETDYQESVaIBBAS0sLtm/fjr/85S9KpbaiogITJkzQOzwioqzxu9/9Ds8++yxeeukl5OTkoLS0FGvXrsX999+vd2hERJmOyQYiomSRZRnt7e1wOBxhFd3q6mrce++9eodHRJSRRML32WefxalTp5SE77JlyzBx4kS9wyMiyhZMNhARjYaPP/4YBw4cQENDA/r7+9kRGRFRgqk9yrZ69WoUFRXpHRoRUTZisoGIaDRxiDUiosQRnfQ+99xzeOedd9hJLxFR6mCygYhIL+fOncPBgwexc+dODAwMKK0dfvSjH2Hs2LF6h0dElLJEK4ZDhw7h2rVrKCkpwfr16/HQQw/pHRoREd3AZAMRkd6uXLmCY8eOweFwoL29HVOmTEFlZSVWr16N/Px8vcMjIkoJFy5cgMvlQkNDA95++23MnDkTK1asQFVVFSZPnqx3eEREFI7JBiKiVNLT04N9+/Zh7969OH/+PBYtWgSz2Yz58+fDYDDoHR4R0ah77733cODAATgcDly8eBFGoxFmsxkLFizQOzQiIoqOyQYiolR0+fJltLW1weFw4Pjx45g+fTqeeuopVFZW4rbbbtM7PCKipLp06RI8Hs+g78BVq1bh9ttv1zs8IiIaGpMNRESp7r333kNDQwP27duHL774gnf1iChjnT59Gs8//zyamppw4cIFtu4iIkpfTDYQEaUL8bzy7t270d3djXvvvRdmsxmrVq1ir+tElLYi+6256667UFVVhR//+Mfst4aIKH0x2UBElI5Cx5MfO3YsTCYTrFYrZs2apXdoRESafPDBB2hqasK+ffvw6aefYu7cuRyRh4goczDZQESUzoLBIJqbm7Fjxw68++67yhjzy5Ytw8SJE/UOj4gozLVr1/Db3/4W27dvx8svv4yvfe1rqKiowOrVq1FYWKh3eERElDhMNhARZYqTJ09i+/btcLvdyMnJQWlpKdatW4f77rtP79CIKMudO3cOBw8exHPPPYdz585h3rx5MJvNeOKJJzBu3Di9wyMiosRjsoGIKNP4fD7s378fe/bswZkzZ/DII4+guroaixcvxvjx4/UOj4iyxPXr19HR0QGHw4EXX3wRt99+O1asWAGLxYI77rhD7/CIiCi5mGwgIspUrOgTkR4++ugjvPDCC2hoaEBfXx/mz58Ps9nMhCcRUXZhsoGIKBsMDAzA6XSyCTMRJUVocvOll15SHuWqrq7Gvffeq3d4REQ0+phsICLKJteuXcMrr7yCZ599Fu3t7ZgyZQqWLVuGNWvW4Bvf+Ibe4RFRmvH7/Xj++efhcDjQ29urdFJbUVGBCRMm6B0eERHph8kGIqJs9f7772Pv3r3Yu3cvPvvsMyxcuBDV1dWYP38+DAaD3uERUQoTw+86nU586UtfQllZGZ555hk88MADeodGRESpgckGIqJsd/nyZbS1tcHhcOD48eP41re+hVWrVmHVqlW4/fbb9Q6PiFJEtKF2n3zySeTk5OgdHhERpRYmG4iI6Ka//vWv2L9/PxwOBy5evAij0Qiz2YwFCxboHRoR6US0Yjh48CDGjh0Lk8kEq9WKWbNm6R0aERGlLiYbiIhosEuXLqGlpQXbtm3D22+/jZkzZ2LFihUwm8348pe/rHd4RJRk58+fh9vtxq5du+D1evHQQw/h6aefxtKlSzFp0iS9wyMiotTHZAMREcUm7moeOnQIY8aMgclkgsViwbe//W29QyOiBBN/74cPH8bVq1chSRJbNxER0XAw2UBERNqI57V37tyJP//5z8rz2suWLcPEiRP1Do+IhunSpUvweDzYvn07fv/73+Oee+7BypUrUVVVhcmTJ+sdHhERpScmG4iIKH5qPdGvXbsW999/v96hEZFGoo+WxsZG/POf/1T6aOGINERElABMNhAR0fD5fD6lQ8ne3l7Mnj0b69atQ3l5Ob70pS/pHR4RReDoM0RENEqYbCAiopG7fv06Ojo64HA48OKLL+K2227DypUr8fTTT+POO+/UOzyirNfT04N9+/Zh7969OH/+PBYtWsRWDERElExMNhARUWINDAzA6XRi165dOHv2LObNmwez2YwnnngC48aN0zs8oqxx5coVHDt2DA6HA+3t7ZgyZQqWLVuGZ555Bl//+tf1Do+IiDIbkw1ERJQcoa0djh49ioKCAlRUVGD16tUoLCzUOzyijHX27Fk0NTVh9+7d+OSTT5SE349+9COMHTtW7/CIiCg7MNlARETJ98EHH6CpqQn79u3Dp59+irlz52LdunX44Q9/yCbcRAkQmdzLz8/H8uXLYbVaMW3aNL3DIyKi7MNkAxERjZ7IZt3f/OY3UVlZiaeeegpf+cpX9A6PKO3wsSUiIkpRTDYQEZE+Tp8+jeeff57D7hHFKVqHrGazGXfddZfe4REREQFMNhARkd4uXboEj8eD7du34/e//z3uuecerFy5ElVVVZg8ebLe4RGljI8//hgHDhzAnj17cObMGWWoWZPJhPHjx+sdHhERUSgmG4iIKHWcPHkSDocDhw4dwrVr11BSUoL169fjoYceiqucL774gj++KOVcv34dADBmzJi4Pve73/0Ozz77LF566SXk5OSgtLQU69atw3333ZeMMImIiBKhNb6rHRERURLNnj0be/bswcDAALZv3w6v14vZs2fjP/7jP+BwOPDPf/5TUznV1dXYtm1bkqMl0u7y5ct48skn8fLLL2taPhAIwOFw4L777sN3v/td9Pb2YufOnTh37hz27NnDRAMREaU8tmwgIqKUJlo7OJ1OjB8/HuXl5XjmmWfwwAMPqC5/4cIFFBQU4F//+hfWr1+P+vr6uO8kEyXSp59+ih/84Afo6urCY489ht/85jdRl1U731evXo2ioqJRjJiIiGjE+BgFERGlh0AggAMHDmDHjh348MMPMXv2bJjNZlRUVGDChAnKcrt378batWtx7do1jBkzBosXL8ahQ4dw66236hg9ZaszZ87g+9//Pvr7+/HFF19gzJgx6O3tDRuOMhgMorm5Gc899xzeeecd5dx+8sknkZOTo2P0REREw8bHKIiIKD3k5eWhuroaPT09eO2113DXXXfhmWeewdSpU1FdXY3e3l4AwK5du5Rn469fv462tjZ897vfxSeffKJn+JSF3nnnHRQXFyuJBgAYO3YsmpqaANxoxfD0008r53BRURFOnjyJP/7xjzCbzUw0EBFRWmPLBiIiSlvnzp1DU1MTGhsb8dFHH+Hhhx9GV1fXoOXGjx+PqVOn4rXXXsPdd9+tQ6SUbV577TUsXrwYV65cwdX/z979xbZ13vcf/5wmTgp4DdkAlWq7U7rfgggGuslIsERGuqaWjQb2ehigsGzTKet1oFzywoBb8yaCBMOT4OSCWgKkgz1R2GAQsCjJN9HZbAywhckXtmIgrTksGKItXqkmwUSsBdncrPl3fhfac8y/EiVTIim9XwAh8vx5znOeQzp5vs+/zz4r2uf3+9XR0aF//dd/1Z49e/STn/xEL7/8sr7yla80KLcAANQdwygAAK3vs88+0z/+4z+qv79f//mf/+m1Ihfatm2bHnvsMf3zP/+znnnmmQbkElvFP/zDP6ivr0+u63q9bApZlqWenh4NDw+ru7u7ATkEAGDdMYwCAND6Hn74YX33u9/VvXv3KgYapKXlMHO5nP78z/9cV69e3eAcYitwXVdnz57VX/3VX+nzzz+vGGiQlpa+/OKLLwg0AAA2NYINAIBN4e///u/1+eefL3vM559/rt///veybVujo6MblDNsBZ999pn6+vo0PDy84rGff/65/uVf/kX/8R//sQE5AwCgMQg2AABanuu6+tu//dsVgw3S0qSRX3zxhX7yk5/or//6rzcgd9jsfve73+l73/ueLl26VLU3Q6mHH35YiURinXMGAEDjMGcDsMndvn1bv/71rxudDWBd/du//ZuGhoYkLXVRtyxLX/rSUjzddV25rls1ELFv3z719fXpoYce2rD8YvP47W9/q+HhYX344YcV9z/00EOyLEuWZUmSN4/DF198oa985Su6cOGCtm3btpFZBjbckSNHGp0FABuPCSKBza63t1dXrlxpdDYAAMAWRXUD2JKmHm50DgCsv8OHD2tqaqrR2QCa1ueff67PP/9cjzzyyIZd07IsTUxM0OK3gsnJSR09erQpKyv/+7//qy9/+cuNzgbQtMzvF8DWRLABALDlPfTQQwyjwKoRaAAAoDomiAQAAAAAAHVFsAEAAAAAANQVwQYAAAAAAFBXBBsAAAAAAEBdEWwAAAAAAAB1RbABAIAWNjg4qMHBwUZno6lYllX0qiSbzWpkZGSDc7Z1jIyMKJ/P1y09ntf6Wu551fJ7AoBKCDYAAIA1y+fzTVsBcV1XruuWbc9mszp79qy2b9/uVaCqBWxKK1rNeq+StLCwoGg0KsuyFI1GNTMzU/E4x3EUCAQUCATkOM6K6SYSiYr3bdKxLEuBQECpVMrbd+DAAYVCIWWz2bXf0P/heTX2eVX7HQHAilwAm9rhw4fdw4cPNzobAEpIcicmJhqdjQc2PT3truf/TkxMTKw6fUlVz8nlcq5t2+7t27e9z+Pj464kd2BgoOI5i4uLriR3cXFxdZnfQLlczp2envbem3sy24zx8XHXtm03l8u5uVzOjUQi7ujoaNV07969W7E84/G4K8m9e/du0XHxeNw75vbt2961HuS+eF7N8byW+11Vs5bfL4BNY5JfP7DJEWwAmtNmCDaYimArBRvi8XjFSqo5Z3x8vGqazay0kuq65eWQyWRcSV7F3XXvVzpNJbRQLpdzBwYGKpZntW22bRdti0QiRRXa1eJ5Nc/zItgAYJUmGUYBAECLymazSqVSCgQCFT87juN1mV5YWPCOMd2ppftdrqPRqObn5720K3VDL90Wj8e9bt2F25t1HolsNqtYLKZ9+/ZV3B+PxxUMBou6ly8nn88rlUp5955IJIq6odfyPAqPHRkZ8fZX61JfjW3bFbdHIhHv/a1btyRJO3fu9Lbt2LFDknTnzp2yc8fGxnTq1KmK6cbjcUnS3NycJHn3MzQ0VHRcb2+vYrHYmoZT8Lxa63kBQJlGhzsArC96NgDNSXXo2WB6FZj/nBd+Nq2hpnU0Eol41y09xnTPluS+9957ruve74quCi2thdtKP7uu6w4MDFTt4r5a9ezZYIZ8ZDKZiue4ruu1DJe2HFdKz7Ztr0v74uKia9t2UTf0Wp5H4bmmlf7GjRtVW69rlcvlyrrlm2dc6d5LW7hv3Ljh5blaeZqyun37tjs+Pl5x2IK530ot+SvheTXX86qW7nLo2QBsaQyjADY7gg1Ac6pHsMGks1Llv5ZjKo3hXmta9VTPYIOpbFU7x3WLh4aYwEvhfsNUMAsrbLdv3y7r2l9LGZrx+qXHPEjA5saNG2Xj76uVS+n2xcXFonkBlnvGpkI8MDBQcay/qUSvZSgFz6u5nhfBBgCrxDAKAAAgdXV1SZJisViDc7J+hoeHVzzG5/NpbGxMkpbtTj41NSVJamtr87bt3r1bknT58uVV5cscXzpMpZb8VvPGG2+ov79fPp9v1ee+9dZb6uvrW/G4kZERvfDCC8rlcpKkUChUtnyiuf5avlc8r9o0y/MCgFIEGwAAAAq0tbXp7t27chxH4XC4rEImSRcvXizbZipqtSxPWMgc7/7fEoOFr7VIpVKybVvd3d1F26vNEyDdnyvAcRy9+OKLNV0jFovp4MGD8vl8CoVCchxHk5OTa8rzg+B5tdbzArB1EGwAAACewgnqtrKuri5NT0/LcRxvcr1CpiJYqSV9rWVYOEHnWqXTab377rsVW7or5dlMFPj0009LkgKBgJ544omqE4QawWBQ0v0Ke3t7uyTp5MmTD3wPa8Hzaq3nBWBrINgAAAC8itOhQ4canJP1YyqhlVq+K7FtW+Pj4xW7xx8/flySdO/ePW+bSbe3t3dV+RodHZUkJZNJLw2z2sFqZLNZXb9+vWiFgXQ6rWg0KkleC3hhnj/66KOifcu11he+L211N5XYaq3xAwMDq7oXiefVas8LAEoRbAAAoEWVLttX+NlUggoraqWtumbJwHw+r2QyKdu2iyofpsXXBCLMsnmSvApRYeurqWw169KXTz31lKTyyqspl0qt3seOHatY8Tp48KBs29b58+e9865du6ZIJKKenp6y9JZ7Hi+99JKkpTH/fr9flmWpvb3dqwSbJRbT6XTVe8tmswqHw4rFYkWt3Hv27PECSB0dHRodHdWlS5eUz+eVz+d16dIljY6OqqOjo2ralZw+fVrS/e+Q+W6Y7YZpiX/22We9bbXcj8TzapbnBQBrtpHTUQLYeKxGATQn1WE1ChUsY1npVemYwm137971ZvIfHR0tm50+k8l4+81SeGbJPzOrv1nFYmBgwNvWrEtfmuU8zRKBhceWllGp0qUGTXqjo6PeeePj4xVXE1jpebjuUlmb1RcikUjRco8DAwNuJBKpmAfDrDJQ6VW4SoPr3l9S0rZt98aNG1XTLL2PUjdu3PCuG4lEKqZlVnwoXAWilvtxXZ6X0ejntVK6y2E1CmBLm7Rcd42z2QBoCaalxczEDaA5WJaliYkJHTlypCHXlrTmCe020uTkpI4ePbqqvC53f6b3xZkzZ+qTwQ0UCAQ0PT3d6GysyuDgoPx+f8XyruV+eF4ba7nntZZ/N9by+wWwaUwxjAIAAGwZ4XBYs7OzRUNCWsHc3Jz6+/sbnY1VSafTSqfTCofDZftqvR+e18ZZ7nkBwFoQbAAAYAspnedhq/H5fBobG9P58+dXnDOgWczMzOjxxx8vWxqxmc3Pz+vixYsaGxvzJiM0VnM/PK+NsdzzAoC1ItgAYFPI5/NFy3xthutms1kNDg56E4eZib0exNzcXFGag4ODSqfTymazDSm/Wm3G59soZsm70vebUelSgEZbW5uSyaSuX7/egFytXk9PjzdZYqtwHEfnzp1TW1tb2b7V3g/Pa/0t97yq/Y4AYCUEGwBsCjdv3txU181ms7p3756Ghobkuq7Gx8cVDAZXvbRaocHBQV26dEmhUMhbIu3UqVNaWFho+krnZnu+jeRWWSZvM6nlHn0+X0vOA9Aqzpw5U7HiulY8r/W13PPaCv9mAFgfBBsAtLx8Pq9EIrGprnvv3r2iLrjHjh2TJMVisTWlZ3owXLhwoajFra2tTbZt6/bt2w+W4XW0GZ8vAADAZkewAUBF+XxeqVTK6z5ZqdJV6ZjS8eCpVEqBQEDSUjdNy7IUCAS8tbxrvZ6p+BV2/zfXisfjchxHUnl3z2w26615HggENDMzs6q81fu6tSod62vWei9dP35wcFCDg4PLpjU3N6fh4eFlJyurNLaY57t+zxcAAGDT28iFNgFsvMOHD7uHDx9e9Xm2bbsDAwPe50gkUvTZHDM6Ouq67tL65bZtu7Zte+uW27btrctt1knPZDLeGt+ruZ5ZG3xxcbFiGqqw/rfJ0/j4uOu6S2uMS3Lv3r1bc97qfd21KFzPvXT99YGBgbLnUsqcW2nd9OXwfNf3+UpyJyYmVnXOVjQxMVH2DAC0Bn6/wJY2ya8f2OTWEmwYHx8vq5zevn3btW3b+2wqWKXHSPIqYa5bubJWuq2W6w0MDCxbCax0HZNu6bVNJbeWvK3HdVfDVIDNKx6PrzqNSnlcCc937detFcGG2lBZAVoXv19gS5u0XJeZXoDNrLe3V5I0NTVV8zmBQECO4yw7EVQ0GtXFixeLjsnn8/L7/bJtW9PT05LkdT0vPK50Wy3XMxYWFjQ1NeXNXWDOqXQdk24lruvWlLf1uO5apNNpXblyRcPDwxodHVVfX1/N51a7p+XwfNf/+VqWpe7ubn3jG9+o+Zyt6IMPPtDc3JwOHz7c6KwAWCXz+6W6AWxJUwQbgE1uLcGGWiqn1Y4p3V5Lha/WynAikZDjOIrH4+rs7Fz1dWq5h0rb6n3dtZqfny+7fi1M4CCXy9W8fjrPd/2fL8GG2hBsAFoXwQZgS5uiXxOwya1lGIUZ777cGHRzTOk8AKphzHvptlquZ7quZzKZimksd53SeQ5Wk7f1uO6DqHS9lUxPT69YvqV4vmu/bq3EMIqa0A0baF38foEtbZLVKACUsW1bknTx4kVvFYSFhQVFo1HvmOPHj0taWqLRMMea3hT1vF4wGJQkdXR01Jzu6OioJCmZTHrpmlUEatWo61Zi0hofH1/VebZty7ZtXbx4seoxCwsLRfnj+a7vdQEAADa9Roc7AKyvtfRsMLPtS/cnJoxEIkUtublczludwLR+j4+PF7V6Ly4ueuebFQxyuZy3zZxXy/XM/kwm47733ntlaRS2xJtJFAuvX/jKZDI1563e162VbdtuPB73zsnlchVXnqhlNYrCMi4tV9ddmoSy8Dma6/F81+/5ui49G2pFyyjQuvj9Alsaq1EAm91al75cXFz0lkwcGBio2GV8cXHRHR0d9Spb4+PjXsXOdd2yyli1bbVc7+7du94+c2wkEvEqeKX7jcJlIwuPrzVv9b5urczQB/OKx+PeEo6Fag02uO5SZXt6etpb7lGSt7xlpfzxfNfv+Zr8EGxYGZUVoHXx+wW2NFajADa7tUwQCWD9WZaliYkJHTlypNFZaWqTk5M6evQoE8wBLYjfL7ClTTFnAwAAAAAAqCuCDQAAAJsEk5Wur5GREW9iWADA8gg2AMAGsiyrphewnvL5/Lp+z9Y7fVSWzWZ19uxZbd++3fu3ZHBwsOKxrfzvTiKRqJhfx3EUCAQUCATkOE7Z/mw2651rWZZSqdSy10mn00okEgoEAt71Dhw4oFAopGw2W5+bAYBNjGADAGwg13VregHr6ebNmy2dPsrl83mFw2GdOHFCkUhEuVxO4+PjGh4erhhwcF1Xi4uLkqTFxcWW+XcnnU7r5MmTZdtTqZQSiYSSyaSSyaSuXr2qRCLh7TflI92/98uXL1cNxoyMjGhwcFBf//rX9fOf/9wrn66uLvX39yscDtPDAQBWQLABAIAtJJ/PF1XCWi19VDY2Nqauri51d3dLknw+n44dOyZJGh4ertiK39bWVvS32eXzeV25cqVs+8LCgoLBoPr7++Xz+eTz+RSJRHTy5Eml02lJ0rVr1+Q4jjcha1tbm4aGhjQ8PKyZmZmi9KLRqHK5nJLJpGzbVkdHR9H+7u5u7dq1S2NjY+t0pwCwORBsAACgReTzeaVSKa8beCKRKOrOXalLfOm2eDzudTE327PZrNcFXbrfTT0ajWp+fv6B05ekwcHBqq3IeDDZbFaxWEz79u2ruD8ejysYDK44bMBY6XuWzWaVSqW874vjOLIsS4FAQAsLC2V5GxkZ8faXVuxXY2xsTKdOnSrbfuvWLUnSzp07vW07duyQJN25c0eSdPnyZUlLQRjjm9/8pqTi1ZrMd3RoaKjo2FK9vb2KxWIMpwCAZRBsAACgRYRCIX388cdeN3DHcYq6c5tu8YUymUzR56GhIe+9GbbT3t7ujXOfm5tTX1+fcrmcJKmzs9MLOKw1fayvt99+W5L05JNPVtx/5swZDQwMKBgMei39y1npexYOhxUMBr3vi23bymQychxHr776qpdONptVOBzWrl275LquTp8+rf3799eUh1IzMzN6/vnnK/bCmJ2dlaSiHgjmOBP4qjSHgwkmXLx4UdLSEI3h4WEdOnTIC7hVC5CYsjZlDwAoR7ABAIAWMDMzI8dx9NJLL0laqkz19/fLcRxdu3bN21aqtAt4JYUBgcJu+JFIRNL9itpa05eWghCFgQjUj2m9X+5ZxGIx2batPXv2FPVWKVXL92x6eto73nxfzLVNxb0wLTOco6enR5IqDoVYTjab1fvvv+9dq1ThNUuZ7675Li9379evX5e0dC8m4LZr1y7t379fc3NzRceaQMVy6QHAVkewAQCAFmC6ehdW+Hfv3i3pfhfxeuvq6pK0VFFF8xoeHl7xGJ/P580xsFz3/3p+z8zxpUNtaslvobfeekt9fX2rOqfUiRMnJEmvv/6610PD9LCIx+OS7n/Pzfe+MOB26dKlovRMsIHfBgBUR7ABAIAWUKn11lR4KnURB0q1tbXp7t27ZcMiCtXze2aOf5AVdxzH0YsvvrjsMbZtV91nggXd3d26ceOGPvzwQ/n9fiUSCf3mN7+RtLScZTUm8LBc7wkAQGUEGwAAaAGmQlWpRdpUqNbLeqePjdPV1aXp6Wk5juO16Bdaj+/Zgww1CAQCeuKJJ6pOTipVzrOZqPLpp5/2tvX09Gh6elqu66qvr0+//OUvNTAw4AUUzP1VCsIsF9AAAFRGsAEAgBZw/PhxSdK9e/e8baZS1Nvbuy7XNJXEQ4cOrUv6qA8TNKhUSa7Etm2Nj49XHM5Qz+/Z6OioJCmZTHppmNUparVcrwjz3vR8KMzzRx99VLSvVCqV0uzsbNEwCHN/v/rVr7xtJt+mXEoNDAzUfC8AsNUQbAAAoAUcPHhQtm3r/PnzXgvutWvXFIlEvIn3pPKJ8AontotGo5KKW4JLK35mecR8Pq9kMinbtotaddeaPktfrp+nnnpKUnmwwXxPKvVSOHbsWMWKci3fs8L0zDULr232m0kmh4eH5ff7ZVmW2tvbvUq9WRJzLatTFOro6NDo6KguXbqkfD6vfD6vS5cuaXR0tGjSzHw+r3Q6rWg0qg8//FDT09NFy1v29PRoYGBAg4OD3j1MTk7Ktm1vkkvD9Jx49tlnHyjvALCZEWwAAKAFmAn+bNtWe3u714X8tddeKzrulVdekW3b6uzslOM46u7u9lqyz507J+n+8pRvvvmmQqFQ0fm7d+9WIBCQ3+9XR0eHkslkXdNH/T333HNzZ8a8AAAgAElEQVSS7rfmS/Iq9pKKvi+FhoaGyoYH1PI9M+lKkt/vL/pbuL+trU2ZTMYLakQiEWUyGS8AkMvlFIlE6hKE6uvr06FDh+T3+xUKhdTb21s0qaRlWfL7/bpz544ikYjOnDlTMR1TJoX3XvobkO6XtSl7AEA5y2UBbGBTMy1IZoZxAM3BsixNTEzoyJEjjc6KpPvj35vtfwsmJyd19OjRpstXszE9SKpVoptZIBAoWk6zFQwODsrv97dkeW8kfr/AljZFzwYAAIAWFw6HNTs7WzSspRXMzc2pv7+/0dlYlXQ6rXQ6rXA43OisAEBTI9gAAMAWVzgGv9L4fjQ/M/zh/PnzDzwHwkaZmZnR448/ru7u7kZnpWbz8/O6ePGixsbGiuZ7AACUI9gAAMAWVzgGv/A9WktbW5uSyaSuX7/e6KzUpKenx5vcslU4jqNz586pra2t0VkBgKb3cKMzAAAAGovx1JuHz+djHoF1RNkCQO3o2QAAAAAAAOqKYAMAAAAAAKgrgg0AAAAAAKCuCDYAAAAAAIC6ItgAAAAAAADqynKZghrY1Hp7e3XlypVGZwMAAGxRVDeALWmKpS+BTe5nP/uZent7G50NAC3q6NGjOn36tPbu3dvorAAAgBZCzwYAAFCVZVmamJjQkSNHGp0VAADQOqaYswEAAAAAANQVwQYAAAAAAFBXBBsAAAAAAEBdEWwAAAAAAAB1RbABAAAAAADUFcEGAAAAAABQVwQbAAAAAABAXRFsAAAAAAAAdUWwAQAAAAAA1BXBBgAAAAAAUFcEGwAAAAAAQF0RbAAAAAAAAHVFsAEAAAAAANQVwQYAAAAAAFBXBBsAAAAAAEBdEWwAAAAAAAB1RbABAAAAAADUFcEGAAAAAABQVwQbAAAAAABAXRFsAAAAAAAAdUWwAQAAAAAA1BXBBgAAAAAAUFcEGwAAAAAAQF0RbAAAAAAAAHVFsAEAAAAAANQVwQYAAAAAAFBXBBsAAAAAAEBdEWwAAAAAAAB1RbABAAAAAADUFcEGAAAAAABQVwQbAAAAAABAXRFsAAAAAAAAdfVwozMAAACaQyaT0eeff162fXFxUffu3SvatnPnTn35y1/eqKwBAIAWY7mu6zY6EwAAoPH+4i/+QlevXl3xuG3btmlxcVFf/epXNyBXAACgBU0xjAIAAEiSjh07tuIxX/rSl/S9732PQAMAAFgWwQYAACBJ+sEPfrDi0AjXdRUKhTYoRwAAoFURbAAAAJKk7du36/vf/762bdtW9ZhHH31U3//+9zcwVwAAoBURbAAAAJ6XX35Zn332WcV927Zt0w9+8ANt3759g3MFAABaDcEGAADgOXTokP7gD/6g4r5PP/1UL7/88gbnCAAAtCKCDQAAwPPII4+ot7dXjzzySNm+xx57TAcOHGhArgAAQKsh2AAAAIocP35cn3zySdG2bdu2KRgMVgxCAAAAlCLYAAAAiuzbt09f+9rXirZ9+umnOn78eINyBAAAWg3BBgAAUORLX/qSXn755aJVKb72ta/p29/+dgNzBQAAWgnBBgAAUCYYDOrTTz+VtDSPw1/+5V/qS1/ifxsAAEBt+L8GAABQ5s/+7M/U0dEhSfrkk0909OjRBucIAAC0EoINAACgjGVZ+tGPfiRJ+n//7//pmWeeaXCOAABAK3m40RkAgK3m9u3b+pu/+ZtGZwNY0e9+9ztJ0pe//GX19vY2ODfAyvbu3auf/exnjc4GAED0bACADffrX/9aV65caXQ2sEl88MEH6/Z9euyxx+T3+/WHf/iH65L+Rpubm9Pc3Fyjs4F1Mjc3p9u3bzc6GwCA/0PPBgBokKmpqUZnAZvA5OSkjh49um7fp+vXr+vAgQPrkvZGM70z+O1tTvS+AYDmQs8GAABQ1WYJNAAAgI1FsAEAAAAAANQVwQYAAAAAAFBXBBsAAAAAAEBdEWwAAAAAAAB1RbABAABIkgYHBzU4ONjobDStbDarkZGRRmdj0xoZGVE+n290NgAAdUKwAQAANIV8Pi/LshqdjYqy2azOnj2r7du3y7IsWZZVNTBj9he+WkUikaiYX8dxFAgEFAgE5DhO2f5sNuuda1mWUqnUstdJp9NKJBIKBALe9Q4cOKBQKKRsNlufmwEANBTBBgAAIEkaGhrS0NBQw65/8+bNhl17Ofl8XuFwWCdOnFAkElEul9P4+LiGh4crBhxc19Xi4qIkaXFxUa7rbnSW1ySdTuvkyZNl21OplBKJhJLJpJLJpK5evapEIuHtN+Uj3b/3y5cvVw3GjIyMaHBwUF//+tf185//3Cufrq4u9ff3KxwO08MBADYBgg0AAKDh8vl8UQW2mYyNjamrq0vd3d2SJJ/Pp2PHjkmShoeHK7bit7W1Ff1tdvl8XleuXCnbvrCwoGAwqP7+fvl8Pvl8PkUiEZ08eVLpdFqSdO3aNTmOoyNHjkhauuehoSENDw9rZmamKL1oNKpcLqdkMinbttXR0VG0v7u7W7t27dLY2Ng63SkAYKMQbAAAAMpms0qlUgoEAhU/O44jy7IUCAS0sLDgHWO610v3u+BHo1HNz897aVcaTlC6LR6Pe93zC7c3eh6JbDarWCymffv2Vdwfj8cVDAZXHDZg5PN5pVIp7x4TiUTRsIFayr3w2JGREW9/acV+NcbGxnTq1Kmy7bdu3ZIk7dy509u2Y8cOSdKdO3ckSZcvX5a0FIQxvvnNb0qSpqamvG3mOQ4NDRUdW6q3t1exWIzhFADQ4gg2AAAAhcNhBYNBr8Jf+Hlubk62bSuTychxHL366quSpPb2dm8M/9zcnPr6+pTL5SRJnZ2dXsDBDCkolMlkij4XDt9wXbdphh68/fbbkqQnn3yy4v4zZ85oYGBAwWDQa+lfTigU0scff+wNN3Acp2jYQC3lLi0FGsLhsHbt2iXXdXX69Gnt37+/pjyUmpmZ0fPPP1+xF8bs7KwkFfVAMMeZ70qlORxMMOHixYuSloZoDA8P69ChQ15QqlqAxJS1KXsAQGsi2AAAADQ9PV31sxk+YCqcpgJZGBAoHGIQiUQk3a+EVqrElnafr6bR80iY1vvl8huLxWTbtvbs2VPUo6PUzMyMHMfRSy+9JGmpXPr7++U4jq5duyaptnIvTMsM5+jp6ZGkikMhlpPNZvX+++971ypVeM1S5vma573cvV+/fl3S0r2YoNSuXbu0f/9+zc3NFR1rAhXLpQcAaH4EGwAAQF11dXVJWqqEt7rh4eEVj/H5fN4cA8t1/zdDCgqDL7t375Z0fyhCrczxpcNRaslvobfeekt9fX2rOqfUiRMnJEmvv/6610PD9LCIx+OS7n8XzHejMCh16dKlovRMsGEzfH8AYCsj2AAAAPCA2tradPfu3bJhEYUq9RIwFetKQxGWY443Q04KX6tJ48UXX1z2GNu2q+4zwYLu7m7duHFDH374ofx+vxKJhH7zm99IWlrOshoTeFiu9wQAoHURbAAAAOvCVEa3iq6uLk1PT8txHK9Fv5CpuFfq+bDWsnqQoQaBQEBPPPFE1Qk8pcp5NhNVPv300962np4eTU9Py3Vd9fX16Ze//KUGBga8gIK5v0pBmOUCGgCA1kWwAQAA1JWpAB86dKjBOXlwJmhQqZJciW3bGh8frzic4fjx45Kke/fuedtMur29vavK1+joqCQpmUx6aZjVKWq1XK8I8970fCjM80cffVS0r1QqldLs7GzRMAhzf7/61a+8bSbfplxKDQwM1HwvAIDmQ7ABAACULb9Y+NlUCgsr3KWt82bpx3w+r2QyKdu2i1qsSycRLJwUMBqNSipuRTeV5kYvffnUU09JKg82mPuv1Evh2LFjFSvKBw8elG3bOn/+vHfetWvXFIlEvAkeay13M8nk8PCw/H6/LMtSe3u7V6k3S2KuZXWKQh0dHRodHdWlS5eUz+eVz+d16dIljY6OFk2amc/nlU6nFY1G9eGHH2p6erpoecuenh4NDAxocHDQu4fJyUnZtu1NcmmYnhPPPvvsA+UdANBYBBsAAIDa29uL3hd+9vv9RX9Lj5eWJjoMBALy+/3q6OhQMpks2v/KK6/Itm11dnbKcRx1d3d7vQDOnTsn6f7yl2+++aZCoVB9b3CNnnvuOUn3W/MleRV7aakcCocfGENDQ2XDA8xEkrZtF5332muvecfUWu5tbW3KZDJeUCMSiSiTyXgBgFwup0gkUpdATV9fnw4dOiS/369QKKTe3t6iSSUty5Lf79edO3cUiUR05syZiumYMim899LviXS/rE3ZAwBak+U2y0LWALBFTE5O6ujRo6uayA2optHfJ1NpbIXvs2n1N6tC1Mr0sqhWiW5mgUCgbFnTZjc4OCi/37/q8l7r8wUArIspejYAAAAsIxwOa3Z2tmjoRyuYm5tTf39/o7OxKul0Wul0WuFwuNFZAQA8IIINAABgTUrnediszPCH8+fPP/AcCBtlZmZGjz/+uLq7uxudlZrNz8/r4sWLGhsbK5rvAQDQmgg2AECLymazSqVSCgQCjc4KtqjSeR42s7a2NiWTSV2/fr3RWalJT0+PN7llq3AcR+fOnVNbW1ujswIAqAOCDQDQos6ePatgMCjHcRqdlTXJ5/Oam5tTIpGoGjBZWFhQNBqVZVmKRqOamZlZ9XUsy6r6GhkZkeM4NS9riGLVlk3crHw+X0vO29Aqzpw5Q6ABADYRgg0A0KIuXLjQ6Cw8kHg8rn/6p3/SyZMnKwZMzFJ6Fy5cUC6X0wsvvKD9+/evOrjiuq4WFxe9z7lczqscHzhwQIlEQqFQaFMPAwAAANhoBBsAAA0xNDTkLXVYyc2bN72lA30+n44dOyZJaxo2UthaWjgWvKurS2NjY5KWJgGkhwMAAEB9EGwAgBaRz+eVSqVkWZYCgYDm5+crHpfNZjUyMuIdZ4YelM7x4DiOd8zCwkJRGub8RCKhbDbrLW+40jXqyQQaSkUikaLPg4ODGhwcXPN12tradPr0aTmOo5s3bxbt2yxlCQAAsNEINgBAiwiFQpqdnVUul9P09LR+8YtflB2TzWYVDoe1a9cuua6r06dPa//+/d5ScmaOh7m5Odm2rUwmI8dx9Oqrr3ppjIyMqLe3V67r6siRI3rzzTdrvsZ6Mr0ODh06VPe0n3nmGUnS1atXvW2buSwBAADWnQsA2FATExPuav/5nZ6ediW57733nrctl8u5korSGh8fL0tbkjswMOC9r7S/cJskd3Fx0fu8uLi4qmusVqU8VXLjxg3Xtm03l8uty3VatSzX8n3aqg4fPuwePny40dnAOuH5AkBTmaRnAwC0ANPiXriUXaV16C9fviypeAUGSRoeHq75WpFIRO3t7UqlUsrn82praytaaaAe11iLN954Q/39/RXvez20Wlkut+oGr6XXlStXdOXKlYbng9f6PV8AQPN4uNEZAACs7OLFizUdZ1ZqcB9gGcKf/vSn+vDDDxUMBiUtrRpRuNxfPa6xWqlUSrZtq7u7e13SN0M0BgYGvG2tVpYTExMPnMZm9/rrr0taei7YfMzzBQA0B4INALAJzc/PF/WCWI2nnnpK09PTSqfTunjxomKxmCQVVZIf9BqrkU6n9e677y67csWDeueddyRJ+/btK9vXKmV55MiRBzp/K5iampJEWW1W5vkCAJoDwygAoAWMjo5K0ooTB5rjksmk11pvVjuolWVZyufz6urq0oULF3T37l2vklyva9Qqm83q+vXrRYGGdDqtaDRa12u88cYbsm1bPT093vbNVpYAAAAbiWADALSAF198UdLSMo9macXCJRJN5full16StDTm3+/3y7Istbe3q7e3V9ls1jveVGzNX0lF++PxuHedr371q4rH496+5a6xWoXXL3xv8hMOhxWLxYrGZe/Zs6doRYpalr6sdh2zsoQkjY2NFZ3TamUJAADQTAg2AEAL6OjoUCaT0a5du/TEE08oGo3qW9/6lmzb1vj4uM6dOydJamtrUyaT8eYeiEQiymQy6ujoUHt7u5ee3+8v+iupaP+pU6c0NTUly7I0NTVV1O1/uWushmVZRdc3lW3j7Nmz3pwGpTo7Ox/4OpZl6fr16+rv79f09LTa2tqKzmulsgQAAGg2lruRM3wBADQ5OamjR49u6ASL2Lz4PtXO9BhhbP/mxPMFgKYyRc8GAAAAAABQVwQbAAAAasDknetrZGSkbO4WAEDrItgAAKibwokcl3th88jn8+v6TNc7/Vpls1mdPXtW27dv977H1SYmbaXv/MLCgqLRqCzLUjQaLZp4tpDjOAoEAgoEAlXnUimUSCQq3rdJx7IsBQIBpVIpb9+BAwcUCoWKJlgFALQugg0AgLpxXbemFzaPmzdvtnT6tcjn8wqHwzpx4oQikYhyuZzGx8c1PDxcMeDguq4WFxclSYuLi037nc/n80qn07pw4YJyuZxeeOEF7d+/vyyYkEqllEgklEwmlUwmdfXqVSUSiarpptNpnTx5smz7yMiIAoGAhoaG5LquhoaGFAwGvd4iXV1d6u/vVzgcpocDAGwCBBsAAMCa5PP5ZSudzZ5+rcbGxtTV1aXu7m5Jks/n07FjxyQtLVta2DpvmNVNSlc5aSY3b96UbduSiu8pEAh4xywsLCgYDKq/v18+n08+n0+RSEQnT55UOp0uSzOfz+vKlSsVrxeLxSQtBRUK/87OznrHdHd3a9euXWVL0QIAWg/BBgAAtqB8Pq9UKuV1808kEkXd1ysNASjdFo/HvVZwsz2bzXpd5aX73emj0ajm5+cfOH1JGhwcrDqEod6y2axisZj27dtXcX88HlcwGKwYcKhkpXLPZrNKpVJe+TmO4w05WFhYKMvbyMiIt7/aEIhqTKChVCQS8d7funVLkrRz505v244dOyRJd+7cKTt3bGxMp06dqphuPB6XJM3NzUmSdz9DQ0NFx/X29ioWizGcAgBaHMEGAAC2oFAopI8//tjr8u84TlH3dTMMoFAmkyn6XFhJNENk2tvbvXH9c3Nz6uvrUy6XkyR1dnZ6AYe1pr/R3n77bUnSk08+WXH/mTNnNDAwoGAwWLGlv9RK5R4OhxUMBr3ys21bmUxGjuPo1Vdf9dLJZrMKh8PatWuXXNfV6dOntX///pryUI3Jw6FDh7xtptdBR0eHt8301igdbjEzM6Pnn3++am8OU1Z79+7V3Nycbt26pcXFRa+Hg2HK2pQ9AKA1EWwAAGCLmZmZkeM4eumllyQtVR77+/vlOI6uXbvmbStVWOGspjAgUDjswLSWmwrqWtOXloIQpa3h68W03i+Xt1gsJtu2tWfPnqLeG6VqKffp6WnveFN+5toXL14sS8sMfejp6ZGkqkMYavHOO+/Itm195zvf8bYVXrNUYbAhm83q/fff9/JczdDQkCKRiPbu3at3331Xjz76aNkxPp9PkpYtSwBA8yPYAADAFjM1NSWpuMK/e/duSdLly5fX5Zqm9dqM228Vw8PDKx7j8/m8OQaW6/5fz3I3x5cOPaklv9W88cYb3twMq/XWW2+pr69vxeNGRkb0wgsveL1dQqFQ2WSQ5vqt9l0BABQj2AAAwBZTqbXaVPBqWdYQ5dra2nT37t2yYRGF6lnu5vh6rfaSSqVk23ZZz4Rq8zpIKuqt8uKLL9Z0jVgspoMHD8rn8ykUCslxHE1OTq4pzwCA5kawAQCALcZUICu1wBdODrge1jv9Rurq6tL09LQcx/EmQyy0HuVej6EG6XRa7777bsWeCZXybCZ2fPrppyUtrV7xxBNPVJ300wgGg5LuB1ja29slqeIymQCA1kewAQCALeb48eOSpHv37nnbTEt8b2/vulzTVIoLJx9sBSZoUKmnQiW2bWt8fLzicIZ6lvvo6KgkKZlMemmY1SlWI5vN6vr160VzYKTTaUWjUUnyeiwU5vmjjz4q2rdc74rC96W9JEzQoVrviYGBgVXdCwCguRBsAABgizl48KBs29b58+e9Futr164pEol4Ew1K91vbTaDALFkoyauMFrZ8l1Z0zXKQ+XxeyWRStm0XVSzXmv5GLn351FNPSSoPNphyq9RL4dixYxUryrWUe2F65pqF1zb7zSSTw8PD8vv9sixL7e3tXtDCLIm53OoUZkWLWCxW1Cthz549XlCoo6NDo6OjunTpkvL5vPL5vC5duqTR0dGaJ/Q0Tp8+Len+98I8b7PdMD0nnn322VWlDwBoLgQbAADYYsyEhrZtq7293evq/tprrxUd98orr8i2bXV2dspxHHV3d3st9+fOnZN0f3nKN998U6FQqOj83bt3KxAIyO/3q6OjQ8lksq7pb4TnnntO0v3WfElexV5SUfkVGhoaqtiSv1K5m3Qlye/3F/0t3N/W1qZMJuMFNSKRiDKZjBcAyOVyikQiywZlzp49W3WuiM7OTu99X1+fDh06JL/fr1AopN7e3pomgyzV09OjGzduaHZ2VpZl6dKlS7px40ZRgEu6X9am7AEArclyG7FoNQBsYZOTkzp69OiaJ3IDCjXj98lUopspT9L9oQpmVYhamR4VZ86cqXue1lsgEChaTrMVDA4Oyu/3r7q81/p8AQDrYoqeDQAAAMsIh8OanZ0tGubRCubm5tTf39/obKxKOp1WOp1WOBxudFYAAA+IYAMAAKibwjkHKs1n0IrM8Ifz588vOwdCM5mZmdHjjz9etpRlM5ufn9fFixc1NjbmTR4JAGhdBBsAAEDdFM45UPi+1bW1tSmZTOr69euNzkpNenp6vMktW4XjODp37pza2toanRUAQB083OgMAACAzaPZ5mmoJ5/P15LzNrQKyhYANhd6NgAAAAAAgLoi2AAAAAAAAOqKYAMAAAAAAKgrgg0AAAAAAKCumCASABpkcnKy0VnAJnD79m1JfJ9q8cEHH0iirDarDz74QN/4xjcanQ0AwP8h2AAADXL06NFGZwGbCN+n2lFWm9fhw4cbnQUAwP+x3M28RhUAAHgglmVpYmJCR44caXRWAABA65hizgYAAAAAAFBXBBsAAAAAAEBdEWwAAAAAAAB1RbABAAAAAADUFcEGAAAAAABQVwQbAAAAAABAXRFsAAAAAAAAdUWwAQAAAAAA1BXBBgAAAAAAUFcEGwAAAAAAQF0RbAAAAAAAAHVFsAEAAAAAANQVwQYAAAAAAFBXBBsAAAAAAEBdEWwAAAAAAAB1RbABAAAAAADUFcEGAAAAAABQVwQbAAAAAABAXRFsAAAAAAAAdUWwAQAAAAAA1BXBBgAAAAAAUFcEGwAAAAAAQF0RbAAAAAAAAHVFsAEAAAAAANQVwQYAAAAAAFBXBBsAAAAAAEBdEWwAAAAAAAB1RbABAAAAAADUFcEGAAAAAABQVwQbAAAAAABAXRFsAAAAAAAAdUWwAQAAAAAA1BXBBgAAAAAAUFcPNzoDAACgOSQSCf32t78t2/7WW2/pv/7rv4q2/fjHP1ZbW9tGZQ0AALQYy3Vdt9GZAAAAjReJRPR3f/d3evTRR6se8+mnn+qrX/2q/vu//1sPP0ybBQAAqGiKYRQAAECSFAwGJUm///3vq74eeughHT9+nEADAABYFsEGAAAgSfrOd76jHTt2LHvMp59+6gUlAAAAqiHYAAAAJEmWZenll1/WI488UvWYnTt3qru7ewNzBQAAWhHBBgAA4AkGg/rkk08q7nvkkUd04sQJWZa1wbkCAACthmADAADwPP3003ryyScr7vvkk08YQgEAAGpCsAEAABT54Q9/qG3btpVtf/LJJ/Unf/InDcgRAABoNQQbAABAkR/+8If67LPPirZt27ZNP/7xjxuUIwAA0GoINgAAgCJ//Md/rD/90z8tmpvhs88+YwgFAACoGcEGAABQ5kc/+pEeeughSUurVDzzzDP6oz/6owbnCgAAtAqCDQAAoEwwGNQXX3whSXrooYf0ox/9qME5AgAArYRgAwAAKLNjxw49//zzsixLX3zxhXp7exudJQAA0EIINgAAgIpCoZBc19V3v/tdff3rX290dgAAQAuxXNd1G50JAGhlhZPoAQBay8TEhI4cOdLobADAZjP1cKNzAACbwenTp7V3795GZwOoyeuvvy5J+ulPf1rTsSdPntT27dvXO1tN6ejRo/y+N7GjR482OgsAsGkRbACAOti7dy8tY2gZU1NTklTTd/bb3/62du7cud5ZalpHjx7l972JEWwAgPXDnA0AAKCqrRxoAAAAa0ewAQAAAAAA1BXBBgAAAAAAUFcEGwAAAAAAQF0RbAAAAAAAAHVFsAEAAKzZ4OCgBgcHG52NppTNZjUyMtLobGxaIyMjyufzjc4GAKAKgg0AAKBl5fN5WZbV6GyUyWazOnv2rLZv3y7LsmRZVtWgjNlf+GpWCwsLikajsixL0WhUMzMzFY9zHEeBQECBQECO46yYbiKRqHjfJh3LshQIBJRKpbx9Bw4cUCgUUjabXfsNAQDWDcEGAACwZkNDQxoaGmrY9W/evNmwa1eTz+cVDod14sQJRSIR5XI5jY+Pa3h4uGLAwXVdLS4uSpIWFxfluu5GZ7km+Xxe6XRaFy5cUC6X0wsvvKD9+/eXBRNSqZQSiYSSyaSSyaSuXr2qRCJRNd10Oq2TJ0+WbR8ZGVEgENDQ0JBc19XQ0JCCwaDXW6Srq0v9/f0Kh8P0cACAJkSwAQAAtKR8Pr9sJbZRxsbG1NXVpe7ubkmSz+fTsWPHJEnDw8NFrfNGW1tb0d9mdPPmTdm2Lan4ngKBgHfMwsKCgsGg+vv75fP55PP5FIlEdPLkSaXT6bI08/m8rly5UvF6sVhM0lJQofDv7Oysd0x3d7d27dqlsbGxOtwhAKCeCDYAAIA1yWazSqVSXmWz9LPjOF7394WFBe8Y0zVeut99PhqNan5+3ku70pCC0m3xeNxrVS/c3sh5JLLZrGKxmPbt21dxfzweVzAYrBhwqCSfzyuVSnn3l0gkioYN1FLmhceOjIx4+6sNgZ9WqnkAACAASURBVKjGBBpKRSIR7/2tW7ckSTt37vS27dixQ5J0586dsnPHxsZ06tSpiunG43FJ0tzcnCR591Pak6a3t1exWIzhFADQZAg2AACANQmHwwoGg16Fv/Dz3NycbNtWJpOR4zh69dVXJUnt7e3eOP65uTn19fUpl8tJkjo7O72AgxlWUCiTyRR9Lqx0uq7bFMMP3n77bUnSk08+WXH/mTNnNDAwoGAwWLGlv1QoFNLHH3/sDbVwHKdo2EAtZS4tBRrC4bB27dol13V1+vRp7d+/v6Y8VGPycOjQIW+b6XXQ0dHhbTO9NUqHW8zMzOj555+v2pvDlNXevXs1NzenW7duaXFx0evhYJiyNmUPAGgOBBsAAMCaTE9PV/1shhCYSufFixclqSggUDjMwLSOmwpppQpoYQV2OY2cR8K03i+X11gsJtu2tWfPnqLeHKVmZmbkOI5eeuklSUtl0t/fL8dxdO3aNUm1lXlhWmboQ09PjyRVHcJQi3feeUe2bes73/mOt63wmqUKgw3ZbFbvv/++l+dqhoaGFIlEtHfvXr377rt69NFHy47x+XyStGxZAgA2HsEGAADQcKa12ozTb1XDw8MrHuPz+bw5Bpbr/j81NSWpOPCye/duSdLly5dXlS9zfOlQlFryW80bb7zhzc2wWm+99Zb6+vpWPG5kZEQvvPCC1/slFAqVTQZprt/q3x0A2GwINgAAAGywtrY23b17t2xYRKFKvQRMxbqW5SQLmePNcJPC11qkUinZtl3WM6HavA6SinqvvPjiizVdIxaL6eDBg/L5fAqFQnIcR5OTk2vKMwBgYxFsAAAATaNwssHNrqurS9PT03Icx5sMsZCpuFfq+bDWcqrHUIN0Oq133323Ys+ESnk2Ezs+/fTTkpZWr3jiiSeqTgJqBINBSfcDLO3t7ZJUcZlMAEDzIdgAAAAazlSCCycbbEUmaFCpp0Iltm1rfHy84nCG48ePS5Lu3bvnbTPp9vb2ripfo6OjkqRkMumlYVanWI1sNqvr168XzYmRTqcVjUYlyeuxUJjnjz76qGjfcr0rCt+X9pIwQYdqvScGBgZWdS8AgPVFsAEAAKxJ6RKMhZ9Nhbaw0l3aQm+Wf8zn80omk7Jtu6giaVrvTSDCLIEoyavcFrakm4pzI5e+fOqppySVBxvMvVfqpXDs2LGKFeWDBw/Ktm2dP3/eO+/atWuKRCLeBI+1lrmZZHJ4eFh+v1+WZam9vd0LWpglMZdbncKsaBGLxYp6JezZs8cLEnV0dGh0dFSXLl1SPp9XPp/XpUuXNDo6WvMEn8bp06cl3f+emOdvthum58Szzz67qvQBAOuLYAMAAFgT063dvC/87Pf7i/6WHi8tTXYYCATk9/vV0dGhZDJZtP+VV16Rbdvq7OyU4zjq7u72egKcO3dO0v3lL998802FQqH63uAaPPfcc5Lut+ZL8ir20lIZFA4VMIaGhiq25I+Njcm27aLzXnvtNe+YWsu8ra1NmUzGC2pEIhFlMhkvAJDL5RSJRJYN0pw9e7bqXBGdnZ3e+76+Ph06dEh+v1+hUEi9vb01TQZZqqenRzdu3NDs7Kwsy9KlS5d048YNL9BimLI2ZQ8AaA6W2wyLUgNAC7MsSxMTEzpy5EijswLUxLRmm9UONpqpNLfC/4Ks5fdtelicOXNmvbK1bgKBQNmSps1ucHBQfr9/TeXNv98AsG6m6NkAAABQR+FwWLOzs0XDPlrB3Nyc+vv7G52NVUmn00qn0wqHw43OCgCgBMEGAGgC2WxWqVRKgUCg0VkB1lXpPA+bkRn+cP78+WXnQGgmMzMzevzxx8uWsmxm8/PzunjxosbGxrzJIwEAzYNgAwA0gbNnzyoYDFYdD90q8vl8xfHotchmsxocHPQmnTOTwq1G4aR1pa+RkRE5jlPzKgHN7kHKupFK53nYrNra2pRMJnX9+vVGZ6UmPT093uSWrcJxHJ07d05tbW2NzgoAoAKCDQDQBC5cuNDoLNTFzZs313ReNpvVvXv3NDQ0JNd1NT4+rmAwuOpl+VzX1eLiovc5l8t5S+sdOHBAiURCoVBoU7Sor7WsG63akoebkc/na8l5G1rFmTNnCDQAQBMj2AAAqIt8Pq9EIrGmc+/du1fUffvYsWOSpFgstuq0CisfhV2ru7q6NDY2JmlpTH0r93B4kLIGAADYCAQbAKAB8vm8UqmULMtSIBDQ/Px80f5sNivHcRQIBJTP5xWNRouWpCs837IsJRKJsrHw5nxJSiQSsixL0Wi07Fq1pFc4HKHatng87g0DKT12JaXjxE0gwCzTZwwODi67NN9K2tradPr0aTmO4/UM2GplDQAAsBEINgBAA4RCIc3OziqXy2l6elq/+MUvivaHw2EFAgE5jqN///d/VyQS0f/8z/8Unf/xxx97wwYcxylqrW9vb/fOn5ubU19fn3K5nCSps7OzrBK8UnqFQxOMTCZT9HloaMh7/yBd5BcWFhSPx7181dszzzwjSbp69aqkrV3WAAAA68YFADwQSe7ExETNx09PT7uS3Pfee8/blsvlXElu4T/L5nMulys6/8aNG64kd3Fx0dt2+/ZtV5I7Pj5edn6hu3fvupLceDxel/Sq5XmtMpmMl0ZpPldjpXxs9bI+fPiwe/jw4TWdu9Ws9veN1sLzBYB1M/nwOsUwAABVmBb1wpnfl1u2rXTf1NSUpOK5CXbv3i1Junz5sjffQSVdXV2SluZCMBPXPUh69dbR0SHXdZVOp3XlyhXFYjE99thj6uvr25Drb6Wy/uCDDzQ5Oblh12tlt2/fbnQWAABoOZbr0vcSAB6EZVmamJjQkSNHaj5eUlnX99LttR73oOc/yHG1prUW8/Pz6uzsXFN6y+Ujn8/L7/drYGDAG46w1cq6t7dXV65cWfV5wGa0mn+/AQA1m2LOBgBoMbZtS1LF5RsjkUhNaRQeV4/01kNhz496eueddyRJ+/btW/HYzVzWhw8fLluGklf5S1qqjDY6H7zW7/kCANYHwQYA2GCjo6OSpHQ6vabzjx8/LmlpuUjDTC7Y29u77LlmssJDhw7VJb31ZPIwPj5etzSz2azeeOMN2batnp6eFY/fKmUNAABQbwQbAGCDvfjii5KWlnFcWFiQJM3MzHj7o9FoxZZv4+DBg7JtW+fPn/eOu3btmiKRSMUKdCqVkrRUqU0mk7Jt22thX016puXdVKLn5uaK8iwVt9yPjIzUVB6SFAgENDIy4pVHPp9XPB7XwMBA0TwGtSx9aSrvpe/T6bTC4bAkaWxszNu+1coaAABgIxBsAIAN1tHRoUwmo127dumJJ55QNBrVt771Ldm2rfHxcZ07d07t7e3e8YFAoOh8n8+nsbEx2bat9vZ2b+z+a6+9VvF6u3fvViAQkN/vV0dHh5LJ5JrSe+WVV2Tbtjo7O+U4jrq7u4vyLN1fkvHNN99c1bKVfX19isVieuKJJ2RZlsbGxvQXf/EXRUs81sKyLPn9fu+z3++XZVmyLEvXr19Xf3+/pqeniyZo3GplDQAAsBGYIBIAHtBqJ4jcKPWcrBHLa7WyNkM2zOoYqK5Zf9+oD54vAKwbJogEAAAAAAD1RbABADahwnkIlpuTAA+OsgYAAChHsAEANqHCeQgK3280M1/CSq9W1ixljebD5J3ra2RkpGgSWABAcyHYAACbULOsJb8V1rrfTPeyUfL5/LoGmdY7/Vpks1mdPXtW27dv94Jq1VZSaaUA3MLCgqLRqCzLUjQaLVpJp5DjOAoEAgoEAnIcZ8V0E4lExfs26ViWpUAg4K34IkkHDhxQKBSiRxEANCmCDQAAYEPdvHmzpdNfST6fVzgc1okTJxSJRJTL5TQ+Pq7h4eGKAQfXdbW4uChJWlxcbNqgVT6fVzqd1oULF5TL5fTCCy9o//79ZcGEVCqlRCKhZDKpZDKpq1evKpFIVE03nU7r5MmTZdtHRkYUCAQ0NDQk13U1NDSkYDDo9Rbp6upSf3+/wuEwPRwAoAkRbAAAABsmn88vW/Fs9vRrMTY2pq6uLnV3d0taWvL02LFjkqTh4eGi1nnDLMdauCxrs7l586Zs25ZUfE+FS8YuLCwoGAyqv79fPp9PPp9PkUhEJ0+eVDqdLkszn8/rypUrFa8Xi8UkLQUVCv/Ozs56x3R3d2vXrl0aGxurwx0CAOqJYAMAAKhJPp9XKpXyuvonEomiLuyVhgGUbovH415LuNmezWa97vLS/S710WhU8/PzD5y+JA0ODlYdxlBP2WxWsVhM+/btq7g/Ho8rGAxWDDhUslKZZ7NZpVIpr+wcx/GGHCwsLJTlbWRkxNtfbQhENSbQUCoSiXjvb926JUnauXOnt23Hjh2SpDt37pSdOzY2plOnTlVMNx6PS5Lm5uYkybufoaGhouN6e3sVi8UYTgEATYZgAwAAqEkoFNLHH3/sdft3HKeoC7sZClAok8kUfS6sKJp5Ltrb272x/XNzc+rr61Mul5MkdXZ2egGHtaa/kd5++21J0pNPPllx/5kzZzQwMKBgMFixpb/USmUeDocVDAa9srNtW5lMRo7j6NVXX/XSyWazCofD2rVrl1zX1enTp7V///6a8lCNycOhQ4e8babXQUdHh7fN9NYoHW4xMzOj559/vmpvDlNWe/fu1dzcnG7duqXFxUWvh4NhytqUPQCgORBsAAAAK5qZmZHjOHrppZckLVUg+/v75TiOrl275m0rVVjprKYwIFA49MC0mJtK6lrTl5aCEKUt4uvBtN4vl69YLPb/2bvf2Dbu+47jn0vipKiHkMlQyo1aZX8CCx660UjQRF6LeJaMZXZ7NNBakuWESQtQBvUggDIL2CKIMAxxdh5IqwEXsCbqiUHAoiU/sW6JnygCrAcNE8Cd+cAPKqBeqWXBxAcLuTxakpZ74N2ZR1ISJZ/MP36/AMHi3e/uvvyRNnzf+/2+P5mmqf3797tGbpSrpc8XFhac9nbf2deempqqOJc99aG7u1uS1p3CUItbt27JNE29+uqrzrbSa5YrTTbkcjn99re/dWJez/j4uKLRqA4cOKA7d+7oqaeeqmjj8/kkacO+BAA8fCQbAADApubn5yW5b/j37dsnSbpy5cqOXNN+gm3P3W8G8Xh80zY+n8+pMbDR8H8v+9xuXz7tpJZ413PhwgWnNsNWXb9+XYODg5u2m5yc1MGDB52RLuFwuKIYpH39ZvqeAMCjgGQDAADYVLUn1vZNXi1LG8ItEAjo9u3bFdMiSnnZ53Z7r5aeTaVSMk2zYmTCenUdJLlGqrz22ms1XWNkZERHjhyRz+dTOByWZVmam5vbVswAgIeLZAMAANiUfRNZ7Sl8aYHAnbDT56+XYDCohYUFWZblFEMstRN97sVUg0wmozt37lQdmVAtZruw44svvijp3uoVzz///LoFP20DAwOS7idY2traJKnqMpkAgMZDsgEAAGzq5MmTkqS7d+862+yn8b29vTtyTfvGuLQAYaOzkwbVRipUY5qmZmdnq05n8LLPp6enJUnJZNI5h706xVbkcjktLi666l9kMhkNDQ1JkjNioTTmzz77zLVvo9EVpb+Xj5Kwkw7rjZ4YGxvb0nsBAOwskg0AAGBTR44ckWmaOnfunPPU+saNG4pGo06xQen+E3c7UWAvWyjJuSEtffpdfrNrLwlZKBSUTCZlmqbr5nK7539YS1/u3bvXib+U3WfVRimcOHGi6o1yLX1eej77mqXXtvfbRSbj8bj8fr8Mw1BbW5uTtLCXxNxodQp7RYuRkRHXqIT9+/c7CaGOjg5NT0/r8uXLKhQKKhQKunz5sqanp2su5mkbHh6WdP87YX/W9nabPXLi5Zdf3tL5AQA7i2QDAADYlF3U0DRNtbW1OcPd33vvPVe7d999V6ZpqrOzU5Zlqaury3l6f/bsWUn3l6e8ePGiwuGw6/h9+/YpFArJ7/ero6NDyWTS0/PvtFdeeUXS/af5kpwbe0muvis1Pj5e9Un+Zn1un1eS/H6/68/S/YFAQNls1klqRKNRZbNZJwGQz+cVjUY3TMicOXNm3VoRnZ2dzu+Dg4M6evSo/H6/wuGwent7ayoGWa67u1sffvihbt68KcMwdPnyZX344Yeu5JZ0v6/tvgcANAaj+LAXoAaAFmMYhq5evaq+vr56hwLUxH6aba920AjsG+lG+2/Jdv5+26MpTp8+vVNh7ZhQKORaTrMZxGIx+f3+bfU3/34DwI6ZZ2QDAACAhyKRiG7evOma4tEM0um0RkdH6x3GlmQyGWUyGUUikXqHAgAoQ7IBAADUVWndgWo1DZqNPf3h3LlzG9ZAaCRLS0t69tlnK5aybGQrKyuamprSzMyMUzwSANA4SDYAAIC6Kq07UPp7MwsEAkomk1pcXKx3KDXp7u52ils2C8uydPbsWQUCgXqHAgCo4ol6BwAAAB5tjVanwSs+n68p6zY0C/oWABobIxsAAAAAAICnSDYAAAAAAABPkWwAAAAAAACeItkAAAAAAAA8RYFIAPDAL37xC83Pz9c7DKAm6XRaktTb21vnSJoDf78BANg6o9iqJaAB4CHhhg2t7MMPP9T3vve9llmSEij393//9zpw4EC9wwCAVjNPsgEAAKzLMAxdvXpVfX199Q4FAAA0j3lqNgAAAAAAAE+RbAAAAAAAAJ4i2QAAAAAAADxFsgEAAAAAAHiKZAMAAAAAAPAUyQYAAAAAAOApkg0AAAAAAMBTJBsAAAAAAICnSDYAAAAAAABPkWwAAAAAAACeItkAAAAAAAA8RbIBAAAAAAB4imQDAAAAAADwFMkGAAAAAADgKZINAAAAAADAUyQbAAAAAACAp0g2AAAAAAAAT5FsAAAAAAAAniLZAAAAAAAAPEWyAQAAAAAAeIpkAwAAAAAA8BTJBgAAAAAA4CmSDQAAAAAAwFMkGwAAAAAAgKdINgAAAAAAAE+RbAAAAAAAAJ4i2QAAAAAAADxFsgEAAAAAAHiKZAMAAAAAAPAUyQYAAAAAAOApkg0AAAAAAMBTJBsAAAAAAICnSDYAAAAAAABPGcVisVjvIAAAQP29+eab+rd/+zfXtv/4j//QH//xH+ub3/yms23Xrl3613/9Vz333HMPO0QAANAc5p+odwQAAKAxdHZ2KplMVmwvFAqu13/xF39BogEAAGyIaRQAAECS9MYbb8gwjA3b7Nq1Sz/72c8eTkAAAKBpkWwAAACSpOeff14vvvjihgmHr7/+Wr29vQ8xKgAA0IxINgAAAMebb76pxx9/vOq+xx57TF1dXfqTP/mThxsUAABoOiQbAACA48SJE/rDH/5Qdd9jjz2mN9988yFHBAAAmhHJBgAA4AgEAjp48GDV0Q3FYlE/+clP6hAVAABoNiQbAACASzgcVvnK2I8//rgOHz6sQCBQp6gAAEAzIdkAAABcfvrTn+qJJ9yrYxeLRb3xxht1iggAADQbkg0AAMDl6aef1pEjR1wJhyeeeEKhUKiOUQEAgGZCsgEAAFR444039Pvf/17SvUTDsWPH9PTTT9c5KgAA0CxINgAAgAo//vGP9c1vflOS9Pvf/16vv/56nSMCAADNhGQDAACo8I1vfEM//elPJUm7d+/W3/3d39U5IgAA0Eye2LwJAMBLn376qX71q1/VOwxgU9/5znckSd///vd1/fr1OkcDbO673/2uDhw4UO8wAACSjGL52lYAgB01Nzen/v7+eocBAC3n+PHjmp+fr3cYAABpnpENAFAn5HrhBTt5tVPfp3/6p3/SP/7jP+rxxx/fkfM/TL29vZLEzWiLsj9fAEBjoGYDAABY1z/8wz+0RKIBAAA8XCQbAADAup54gkGQAABg60g2AAAAAAAAT5FsAAAAAAAAniLZAAAAAAAAPEWyAQAAAAAAeIpkAwAAkCTFYjHFYrF6h9GwcrmcJicn6x1Gy5qcnFShUKh3GAAAj5BsAAAADaFQKMgwjHqHUVUul9OZM2e0e/duGYYhwzDWTczY+0t/GtXq6qqGhoZkGIaGhoa0tLRUtZ1lWQqFQgqFQrIsa9PzJhKJqu/bPo9hGAqFQkqlUs6+w4cPKxwOK5fLbf8NAQAaBskGAAAgSRofH9f4+Hjdrr+8vFy3a2+kUCgoEonorbfeUjQaVT6f1+zsrOLxeNWEQ7FY1NramiRpbW1NxWLxYYdck0KhoEwmo0uXLimfz+vgwYPq6empSCakUiklEgklk0klk0l98MEHSiQS6543k8no1KlTFdsnJycVCoU0Pj6uYrGo8fFxDQwMOKNFgsGgRkdHFYlEGOEAAC2AZAMAAKi7QqGw4Q1sPc3MzCgYDKqrq0uS5PP5dOLECUlSPB53PZ23BQIB15+NaHl5WaZpSnK/p1Ao5LRZXV3VwMCARkdH5fP55PP5FI1GderUKWUymYpzFgoFXbt2rer1RkZGJN1LKpT+efPmTadNV1eX2tvbNTMz48E7BADUE8kGAACgXC6nVCrl3GiWv7Ysyxn6vrq66rSxh8VL94fODw0NaWVlxTl3tekE5dsmJiacJ+ql2+tdRyKXy2lkZESHDh2qun9iYkIDAwNVEw7VFAoFpVIp5z0mEgnXtIFa+r207eTkpLN/vSkQ67ETDeWi0ajz+69+9StJ0nPPPeds+/a3vy1J+uSTTyqOnZmZ0dtvv131vBMTE5KkdDotSc77KR9N09vbq5GREaZTAECTI9kAAAAUiUQ0MDDg3PCXvk6n0zJNU9lsVpZl6fz585KktrY2Zw5/Op3W4OCg8vm8JKmzs9NJONhTCkpls1nX69IbzmKx2DBTDz7++GNJ0gsvvFB1/+nTpzU2NqaBgYGqT/rLhcNhffHFF85UC8uyXNMGaul36V6iIRKJqL29XcViUcPDw+rp6akphvXYMRw9etTZZo866OjocLbZozXKp1ssLS3pBz/4wbqjOey+OnDggNLptH71q19pbW3NGeFgs/va7nsAQHMi2QAAALSwsLDua3v6gH3DOTU1JUmuhEDpFAP7ybh9M1rt5rP05nUj9a4jYT+93yjekZERmaap/fv3u0Z0lFtaWpJlWTp27Jike/0yOjoqy7J048YNSbX1e+m57KkP3d3dkrTuFIZa3Lp1S6Zp6tVXX3W2lV6zXGmyIZfL6be//a0T83rGx8cVjUZ14MAB3blzR0899VRFG5/PJ0kb9iUAoPGRbAAAAJ6yn1Tbc/SbWTwe37SNz+dzagxsNPx/fn5ekjv5sm/fPknSlStXthSX3b58Okot8a7nwoULTm2Grbp+/boGBwc3bTc5OamDBw86I2DC4XBFMUj7+q3w/QGARxnJBgAAgAcUCAR0+/btimkRpaqNErBvrGtZTrKU3d6eclL6sx2pVEqmaVaMTFivroMk1wiW1157raZrjIyM6MiRI/L5fAqHw7IsS3Nzc9uKGQDQ2Eg2AACAHVFaaPBREAwGtbCwIMuynGKIpewb92ojH7bbV15MNchkMrpz507VkQnVYrYLO7744ouS7q1e8fzzz69bCNQ2MDAg6X6Cpa2tTZKqLpMJAGh+JBsAAICn7Bvg0kKDzcpOGlQbqVCNaZqanZ2tOp3h5MmTkqS7d+862+zz9vb2bimu6elpSVIymXTOYa9OsRW5XE6Li4uuuhiZTEZDQ0OS5IxYKI35s88+c+3baHRF6e/loyTspMN6oyfGxsa29F4AAI2FZAMAAKhYfrH0tX0zW3rDXf503l76sVAoKJlMyjRN102k/eTeTkTYyx9Kcm5sS5+i2zfN9V76cu/evZIqkw32+682SuHEiRNVb5SPHDki0zR17tw557gbN24oGo06BR5r7Xe7yGQ8Hpff75dhGGpra3OSFvaSmButTmGvaDEyMuIalbB//34nUdTR0aHp6WldvnxZhUJBhUJBly9f1vT0dM1FPm3Dw8OS7n9X7O+Avd1mj5x4+eWXt3R+AEBjIdkAAACcIe3276Wv/X6/68/y9tK9QoehUEh+v18dHR1KJpOu/e+++65M01RnZ6csy1JXV5czCuDs2bOS7i9/efHiRYXDYW/f4Da98sorku4/zZfk3NhL9/qhdKqAbXx8vOqT/JmZGZmm6Truvffec9rU2u+BQEDZbNZJakSjUWWzWScBkM/nFY1GN0zUnDlzZt1aEZ2dnc7vg4ODOnr0qPx+v8LhsHp7e2sqBlmuu7tbH374oW7evCnDMHT58mV9+OGHTqLFZve13fcAgOZkFBtlIWsAeETMzc2pv79/24XcgFL1/j7ZN8zN8H22n/rbq0LUyh5lcfr0ac9j2mmhUKhiWdNGF4vF5Pf7t9zf2/18AQA7Yp6RDQAAABuIRCK6efOma+pHM0in0xodHa13GFuSyWSUyWQUiUTqHQoA4AGRbACAJpXL5ZRKpRQKheodCh5R5XUeWpU9/eHcuXMb1kBoJEtLS3r22WcrlrJsZCsrK5qamtLMzIxTPBIA0LxINgBAkzpz5owGBgbWnXPd6AqFgtLptBKJxLoJk1wup1gs5hSuswvLbUVp4bvyn8nJSVmWVfNKA3Arr/PQygKBgJLJpBYXF+sdSk26u7ud4pbNwrIsnT17VoFAoN6hAAA8QLIBAJrUpUuX6h3CA5mYmND777+vU6dOVU2Y5HI53b17V+Pj4yoWi5qdndXAwMCWl/YrFotaW1tzXufzeWd5vsOHDyuRSCgcDrf0k/mdst5yh63K5/M1Zd2GZnH69GkSDQDQQkg2AADqYnx83Fl9oJq7d++6hoCfOHFCkjQyMrLla5XewJQOzw4Gg5qZmZF0b14+IxwAAAC8QbIBAJpEoVBQKpWSYRgKhUJaWVmp2i6Xy2lyctJpt7S05GwvrfFgWZbTxl7X3mYfn0gklMvlKpb2W+8aXiqfa24nAuyl/myxWGzD5f02EwgENDw8LMuytLy87NrXKn0JAADwsJFsAIAmEQ6HdfPmTeXzeS0sLOjXv/51RZtcLqdIJKL2nRrFbwAAIABJREFU9nYVi0UNDw+rp6fHqe5u13hIp9MyTVPZbFaWZen8+fPOOSYnJ9Xb26tisai+vj5dvHix5mvslNXVVU1MTDj94LWXXnpJkvTBBx8421q1LwEAAB6KIgDgobp69Wpxq//8LiwsFCUVf/Ob3zjb8vl8UZLrXLOzsxXnllQcGxtzfq+2v3SbpOLa2przem1tbUvX2KpqMZXKZrNOG0nFiYmJHblOs/bldr5Pj6rjx48Xjx8/Xu8wsEP4fAGgocw9sdPJDADAg7OfuJdWl6+2NNyVK1ckqWKofjwe37A+QqloNKq2tjbNzs7qyJEjCgQCruJ/XlxjKzo6OlQsFpXJZHTt2jWNjIzo6aef1uDgoOfXKtVsfdnb27ul9o+idDotib5qVel0uqmW+gSAVsc0CgBoAlNTUzW1s1d1KJatElDcwkoB77zzjkzT1MDAgPx+f8XqD15cYzuCwaAzheLUqVOenrtaPYhW7ksAAICdxsgGAGhBKysrrlEQW7F3714tLCwok8loamrKWf2hfMm/B7nGdu3U9W7duiVJOnToUMW+ZunL+fn5Bzr+UWCPaKCvWhMjVgCgsTCyAQCawPT0tCRtWjjQbpdMJp2n9fZqB7UyDEOFQkHBYFCXLl3S7du3XctNenGN7bKvNzs769k5c7mcLly4INM01d3d7Wxv9b4EAADYSSQbAKAJvPbaa5LuLfNoL61YukTi0NCQJOnYsWOS7s359/v9MgxDbW1t6u3tVS6Xc9rbN7b2n5Jc+ycmJpzrPPPMM85KEJtdY6tKr1/6uySFQiFNTk46cRQKBU1MTGhsbEwnTpxw2tWy9OV617FXlpCkmZkZ1zHN1pcAAACNhGQDADSBjo4OZbNZtbe36/nnn9fQ0JC+973vyTRNzc7O6uzZs5KkQCCgbDbr1B6IRqPKZrPq6OhQW1ubcz6/3+/6U5Jr/9tvv635+XkZhqH5+XnXsP+NrrEVhmG4rm/fbNsGBwc1MjKi559/XoZhaGZmRj/60Y+2XDhxvesYhqHFxUWNjo5qYWFBgUDAdVwz9SUAAECjMYpUoQKAh2pubk79/f0UAYQn+D7VjpoNrY3PFwAayjwjGwAAAAAAgKdINgAAAGwDxTy3b3JysqJOCwCgtZBsAAB4xq6FsNkPWkehUNjRz3Snz79duVxOZ86c0e7du53v9XqFSpvt70Amk3HFahegtRUKBaXTaSUSCYVCoarnWF1d1dDQkHN8aUFbSTp8+LDC4bCrmCoAoLWQbAAAeKZYLNb0g9axvLzc1OffjkKhoEgkorfeekvRaFT5fF6zs7OKx+NVEw7FYlFra2uSpLW1tYb/O/DJJ5+4Xh89etT1emJiQu+//75OnToly7Iqji8UCspkMrp06ZLy+bwOHjyonp4eV9tgMKjR0VFFIhFGOABAiyLZAAAAtqVQKCiRSDTt+bdrZmZGwWBQXV1dkiSfz+csxxqPx5VKpSqOsVc7KV/1pBHt2bPHlRw0TdO1f3x8fMNVYZaXl51jSvumfBREV1eX2tvbK5adBQC0BpINAAA8ggqFglKplDNUPpFIuIa0VxvyX75tYmLCeVptb8/lcrIsy7mxTCQSzlD6lZWVBz6/JMVisXWnLOy0XC6nkZERHTp0qOr+iYkJDQwMVE04VLPZ55DL5ZRKpZz+tCxLhmEoFAppdXW1IrbJyUlnf/nUhVqsrq4qFAopFospnU5v+XhJFckJWzQardjW29urkZERplMAQAsi2QAAwCMoHA7riy++cIb4W5blGtJuD/svlc1mXa9Ln27bT8Hb2toUCoVkWZbS6bQGBweVz+clSZ2dnU7CYbvnr7ePP/5YkvTCCy9U3X/69GmNjY1pYGBAmUxm0/Nt9jlEIhENDAw4/WmaprLZrCzL0vnz553z5HI5RSIRtbe3q1gsanh4WD09PTXFUMpuH4/HdeDAAYVCoQdOBNjvpXw6hnS/H+1+BQC0DpINAAA8YpaWlmRZlo4dOybp3tD+0dFRWZalGzduONvKdXR0bHru0oRA6TQD+6m2PVJhu+eXNh/Gv5PsegYbxToyMiLTNLV//37XaI5ytXwOCwsLTnu7P+1rT01NVZzLnrLQ3d0tSbp27dqW3p9pmsrn87p9+7bGxsZkWZauX7++pXOUu3XrlkzT1Kuvvlqxz+fzSdKG/QQAaE4kGwAAeMTMz89Lct/w79u3T5J05cqVHblmMBiUdO9GvJnF4/FN2/h8PqcOwUZTBLz8HOz25VNRaom3nM/nUzAY1Pj4uKanp6sWgdyKCxcuaHR01EkslF9Lav7vBQCgEskGAAAeMaVPxG32Td+D3ljinkAgoNu3b1dMiyjl5edgt/d69Ze+vr4H+k6kUimZpumMygAAPDpINgAA8IixC/hVe+JerYifl3b6/I0kGAxqYWFBlmVpYmKiYv9OfA5eT0conQKzVZlMRnfu3NHg4KCnMQEAmgPJBgAAHjEnT56UJN29e9fZZj957+3t3ZFr2jfB1YoENhM7aVBtpEI1pmlqdna26nQGLz+H6elpSVIymXTOYa9O8SAKhcK2vhO5XE6Li4uu2hqZTEZDQ0NV24+NjW07RgBAYyLZAADAI+bIkSMyTVPnzp1znqrfuHFD0WjUKSwo3X+6bicKSpdCtG8aS5/Ol9/Y2ss/FgoFJZNJmabpWhZxu+ev59KXe/fulVSZbLD7sdoohRMnTlS9ma7lcyg9n33N0mvb++0ik/F4XH6/X4ZhqK2tzUkU2EtibrQ6RSqVci2Xubq6quXlZdd3ojyW9foiEoloZGTEVUNi//79Fckme/nOl19+ed24AADNiWQDAACPGLuAoWmaamtrc4oJvvfee6527777rkzTVGdnpyzLUldXl/Ok/uzZs5LuL0958eJFhcNh1/H79u1TKBSS3+9XR0eHksmkp+evh1deeUWS9Nlnnznb7Bt7Sa7+LDU+Pu5KtEi1fQ72eSXJ7/e7/izdHwgElM1mnaRGNBpVNpt1Vq7I5/OKRqMbJml2796tnp4eGYahWCymzz//vCJm+/2WxmAnN2xnzpxZt85DZ2en67Xdj3a/AgBah1FshEWrAeARMjc3p/7+/gcu3AZIjfl9sm88Gykm6f7UBHsViO2yR1icPn36gWN62EKhkGs5zXqLxWLy+/2e9KVXny8AwBPzjGwAAADYgkgkops3b7qmfTSDdDqt0dHReofhyGQyymQyikQi9Q4FALADSDYAAADPlNYYqFa/oBXY0x/OnTu3YQ2ERrK0tKRnn322YZagXFlZ0dTUlGZmZpzlPgEArYVkAwAA8ExpjYHS31tNIBBQMpnU4uJivUOpSXd3t1PcshFYlqWzZ88qEAjUOxQAwA55ot4BAACA1tFodRp2ks/na8q6DY2AfgOA1sfIBgAAAAAA4CmSDQAAAAAAwFMkGwAAAAAAgKdINgAAAAAAAE+RbAAAAAAAAJ5iNQoAqBPDMOodAloI36fa0Vet6/jx4/UOAQDw/0g2AMBD9td//de6evVqvcMAatLf36/h4WEdOHCg3qEAm/rud79b7xAAAP/PKD5KC2IDAIAtMQxDV69eVV9fX71DAQAAzWOemg0AAAAAAMBTJBsAAAAAAICnSDYAAAAAAABPkWwAAAAAAACeItkAAAAAAAA8RbIBAAAAAAB4imQDAAAAAADwFMkGAAAAAADgKZINAAAAAADAUyQbAAAAAACAp0g2AAAAAAAAT5FsAAAAAAAAniLZAAAAAAAAPEWyAQAAAAAAeIpkAwAAAAAA8BTJBgAAAAAA4CmSDQAAAAAAwFMkGwAAAAAAgKdINgAAAAAAAE+RbAAAAAAAAJ4i2QAAAAAAADxFsgEAAAAAAHiKZAMAAAAAAPAUyQYAAAAAAOApkg0AAAAAAMBTJBsAAAAAAICnSDYAAAAAAABPkWwAAAAAAACeItkAAAAAAAA8RbIBAAAAAAB4imQDAAAAAADwFMkGAAAAAADgqSfqHQAAAGgM2WxWv//97yu2r62t6e7du65tzz33nL7xjW88rNAAAECTMYrFYrHeQQAAgPr70Y9+pA8++GDTdrt27dLa2pqeeeaZhxAVAABoQvNMowAAAJKkEydObNrmscce09/+7d+SaAAAABsi2QAAACRJP/nJTzadGlEsFhUOhx9SRAAAoFmRbAAAAJKk3bt368c//rF27dq1bpunnnpKP/7xjx9iVAAAoBmRbAAAAI7XX39dX3/9ddV9u3bt0k9+8hPt3r37IUcFAACaDckGAADgOHr0qP7oj/6o6r6vvvpKr7/++kOOCAAANCOSDQAAwPHkk0+qt7dXTz75ZMW+p59+WocPH65DVAAAoNmQbAAAAC4nT57Ul19+6dq2a9cuDQwMVE1CAAAAlCPZAAAAXA4dOqRvfetbrm1fffWVTp48WaeIAABAsyHZAAAAXB577DG9/vrrrlUpvvWtb+mHP/xhHaMCAADNhGQDAACoMDAwoK+++krSvToOP/vZz/TYY/y3AQAA1Ib/NQAAgArf//731dHRIUn68ssv1d/fX+eIAABAMyHZAAAAKhiGoTfffFOS9Gd/9md66aWX6hwRAABoJk/UOwAAgPTP//zP+uijj+odBuDyP//zP5Kkb3zjG+rt7a1zNECl+fn5eocAAFgHIxsAoAF89NFHSqfT9Q4DLSKdTnvyfXr66afl9/v13e9+14OoGtO1a9f06aef1jsMbNGnn36qa9eu1TsMAMAGGNkAAA2iq6uLp3TwhD0KwYvv0+Liog4fPvzA52lUhmHonXfeUV9fX71DwRbMzc1RRwQAGhwjGwAAwLpaOdEAAAB2DskGAAAAAADgKZINAAAAAADAUyQbAAAAAACAp0g2AAAAAAAAT5FsAAAA64rFYorFYvUOoyHlcjlNTk7WO4ymNDk5qUKhUO8wAAA7iGQDAABoWIVCQYZh1DuMCrlcTmfOnNHu3btlGIYMw1g3KWPvL/1pZJlMxhXr0NCQa3+hUFA6nVYikVAoFKp6jtXVVQ0NDTnHLy0tufYfPnxY4XBYuVxux94HAKC+SDYAAIB1jY+Pa3x8vG7XX15ertu111MoFBSJRPTWW28pGo0qn89rdnZW8Xi8asKhWCxqbW1NkrS2tqZisfiwQ96STz75xPX66NGjrtcTExN6//33derUKVmWVXF8oVBQJpPRpUuXlM/ndfDgQfX09LjaBoNBjY6OKhKJMMIBAFoUyQYAANCQCoWCEolEvcOoMDMzo2AwqK6uLkmSz+fTiRMnJEnxeFypVKrimEAg4Pqzke3Zs0fFYtH5MU3TtX+zBNTy8rJzTGnflI+C6OrqUnt7u2ZmZjx+BwCARkCyAQAAVJXL5ZRKpZybxPLXlmXJMAyFQiGtrq46bSzLctokEglnKP3Kyopz7mpTCsq3TUxMOE/DS7fXs45ELpfTyMiIDh06VHX/xMSEBgYGqiYcqikUCkqlUs77SyQSrqkFtfR5advJyUlnf/nUhVqsrq4qFAopFospnU5v+XhJFckJWzQardjW29urkZERplMAQAsi2QAAAKqKRCIaGBhwbvhLX6fTaZmmqWw2K8uydP78eUlSW1ubQqGQ02ZwcFD5fF6S1NnZ6SQc7GkFpbLZrOt16dNz+yl7vX388ceSpBdeeKHq/tOnT2tsbEwDAwPKZDKbni8cDuuLL75wplpYluWaWlBLn0v3Eg2RSETt7e0qFosaHh5WT09PTTGUstvH43EdOHBAoVDogRMB9nspn44h3e9Hu18BAK2DZAMAAKhqYWFh3df2FIKOjg5J0tTUlCS5EgKl0wzsp9p24qLadAL7XJupZx0Ju57BRrGOjIzINE3t37/fNZqj3NLSkizL0rFjxyTd65PR0VFZlqUbN25Iqq3PS89lT1no7u6WJF27dm1L7880TeXzed2+fVtjY2OyLEvXr1/f0jnK3bp1S6Zp6tVXX63Y5/P5JGnDfgIANCeSDQAAYMcFg0FJ927Em1k8Ht+0jc/nc+oQbDRFYH5+XpI78bJv3z5J0pUrV7YUl92+fCpKLfGW8/l8CgaDGh8f1/T0dNUikFtx4cIFjY6OOomF8mtJzf+9AABUItkAAADgsUAgoNu3b1dMiyhVOjLBZt98b/UG325fWtjRi6knfX19D5RsSKVSMk3TGZUBAHh0kGwAAAAPTbUiga0qGAxqYWFBlmVpYmKiYr9dSLHayIft9pPX0xFKp8BsVSaT0Z07dzQ4OOhpTACA5kCyAQAA7Dj7JrhakcBmYicNqo1UqMY0Tc3OzladznDy5ElJ0t27d51t9nl7e3u3FNf09LQkKZlMOuewV6d4EIVCYcux2NdeXFx01dbIZDIaGhqq2n5sbGzbMQIAGhPJBgAAUFX5Eoylr+0b2tKb7vIn9Pbyj4VCQclkUqZpupZFtJ+Y24mI0qUW7ZvS0qf/9o1zPZe+3Lt3r6TKZIP93quNUjhx4kTVm+kjR47INE2dO3fOOe7GjRuKRqNOgcda+9wuMhmPx+X3+2UYhtra2pxEgb0k5karU6RSKddymaurq1peXnZiKVUaQ7W+iEQiGhkZcdWQ2L9/f0WyyV6+8+WXX143LgBAcyLZAAAAqmpra3P9Xvra7/e7/ixvL90rdhgKheT3+9XR0aFkMuna/+6778o0TXV2dsqyLHV1dTkjAc6ePSvp/vKXFy9eVDgc9vYNbsMrr7wiSfrss8+cbfaNvXSvD+zijKXGx8ddiRbpfiFJ0zRdx7333ntOm1r7PBAIKJvNOkmNaDSqbDbrrFyRz+cVjUY3TNLs3r1bPT09MgxDsVhMn3/+eUXM9vstjcFObtjOnDmzbp2Hzs5O12u7H+1+BQC0DqPYCItWA8Ajzn76aFenBx5Evb9P9o1nM/wXwzAMXb16VX19fTUfY4+wOH369E6FtWNCoVDFkqb1FIvF5Pf7t9yXc3Nz6u/vb4rvGAA8ouYZ2QAAALAFkUhEN2/edE37aAbpdFqjo6P1DsORyWSUyWQUiUTqHQoAYAeQbAAAAJ4pr/PQiuzpD+fOnduwBkIjWVpa0rPPPtswS1CurKxoampKMzMzznKfAIDWQrIBAFpILpdTKpVSKBSqdyh4RJXXeWhVgUBAyWRSi4uL9Q6lJt3d3U5xy0ZgWZbOnj2rQCBQ71AAADuEZAMAtJAzZ85oYGBg3eJsja5QKCidTiuRSNScMEkkElUL8m2ktEJ++c/k5KQsy6p5aUO4FYtF108r8/l8TVm3oRGcPn2aRAMAtDiSDQDQQi5dulTvEB7IxMSE3n//fZ06daqmhEkmk9GpU6e2fJ1isai1tTXndT6fd26ODx8+rEQioXA43LLTAAAAAHYayQYAQMMYHx93ljrcTKFQ0LVr17Z9rdKnqqVzxoPBoGZmZiTdKwTICAcAAICtI9kAAE2sUCgolUrJMAyFQiGtrKxUbZfL5TQ5Oem0W1pacraX1niwLMtps7q66jqHfXwikVAul6uYurDeNXbKzMyM3n777ar7YrGYYrHYts8dCAQ0PDwsy7K0vLzs2teKfQkAAOA1kg0A0MTC4bBu3rypfD6vhYUF/frXv65ok8vlFIlE1N7ermKxqOHhYfX09DhLztk1HtLptEzTVDablWVZOn/+vHOOyclJ9fb2qlgsqq+vTxcvXqz5GjthaWlJP/jBD3Z0zvdLL70kSfrggw+cba3YlwAAADuiCACou+PHjxePHz++pWMWFhaKkoq/+c1vnG35fL4oqVj6z/vs7Gyx/J97ScWxsTHn92r7S7dJKq6trTmv19bWtnSNraoWU+m1p6ena2r7INeptr9Z+nI736dHlaTi1atX6x0Gtujq1avb/nsPAHgo5hjZAABNyn7iXrqcXbX16q9cuSLJvQKDJMXj8ZqvFY1G1dbWplQqpUKhoEAg4FppwItr1Or69esaHBz0/Ly1aKa+vHbt2oarbvBzv2/7+/vrHgc/W/vp7+/f0t8HAMDD90S9AwAAbM/U1FRN7exVHYoPsAzhO++8o//8z//UwMCApHurRpQu+efFNWphWZZee+21Hb2GzS4MOTY25rq+1Bx92dXVpXfeeeeBzvEo6O/v1/DwsA4cOFDvULAFH330kS5cuFDvMAAAGyDZAACPiJWVFdcoiK3Yu3evFhYWlMlkNDU1pZGREUly3SQ/6DVqYRdfrMYwDE+THbdu3ZIkHTp0qGJfM/Tld77zHfX19W37+EdFf3+/Dhw4QF81IZINANDYmEYBAE1qenpakjYtHGi3SyaTztN6e7WDWhmGoUKhoGAwqEuXLun27dvOTbJX16hFsVis+Cnd55VcLqcLFy7INE11d3c721upLwEAAHYSyQYAaFL2dIJYLOYsrVi6ROLQ0JAk6dixY5Luzfn3+/0yDENtbW3q7e1VLpdz2ts3tvafklz7JyYmnOs888wzmpiYcPZtdI2tKr1+6e9bUcvSl+tdx15ZQrq3vGapZutLAACAeiHZAABNqqOjQ9lsVu3t7Xr++ec1NDSk733vezJNU7Ozszp79qwkKRAIKJvNOrUHotGostmsOjo61NbW5pzP7/e7/pTk2v/2229rfn5ehmFofn7eNex/o2tshWEYruvbN9teW+86hmFocXFRo6OjWlhYqFhas5n6EgAAoJ6M4k5X8wIAbMp+aj0/P1/nSNAK+D7VzjAMXb16lZoNTWZubk79/f07XpQWALBt84xsAAAAAAAAniLZAAAAsA0U7ty+ycnJbddkAQA0B5INAIAdZddC2OwHraNQKOzoZ7rT569FLpfTmTNntHv3buc7vF5R0mb6vq+urmpoaEiGYWhoaMhVdLZUJpNxvR+7IO16bROJhEKhkPPeDx8+rHA47CqcCgBoLSQbAAA7qtpylRstYYnmt7y83NTn30yhUFAkEtFbb72laDSqfD6v2dlZxePxqgmHYrGotbU1SdLa2lrDft8LhYIymYwuXbqkfD6vgwcPqqenR5ZlVbT95JNPXK+PHj1a9ZyTk5OKxWLas2ePfvnLXzrvPRgManR0VJFIhBEOANCiSDYAAADPFAoFJRKJpj1/LWZmZhQMBtXV1SVJ8vl8OnHihKR7S5amUqmKY+yVTcpXOGkky8vLMk1Tkvs9hUKhirZ79uxxJQvt40oNDQ0pn88rmUzKNM2KFVW6urrU3t5escQsAKA1kGwAAACS7t3Ip1IpZ2h8IpFwDXOvNg2gfNvExITzJNzensvlZFmWc9OaSCScofcrKysPfH5JisVi605j8FIul9PIyIgOHTpUdf/ExIQGBgaqJhyq2azPc7mcUqmU03eWZckwDIVCIa2urlbENjk56exfbwrEeqolDKR7y6+WWl1dVSgUUiwWUzqdrnqM/VmMj4/L5/Ote83e3l6NjIwwnQIAWhDJBgAAIEkKh8P64osvnGH/lmW5hrnbUwFKZbNZ1+vx8XHnd/upd1tbm0KhkCzLUjqd1uDgoPL5vCSps7PTSThs9/wP08cffyxJeuGFF6ruP336tMbGxjQwMKBMJrPp+Tbr80gkooGBAafvTNNUNpuVZVk6f/68c55cLqdIJKL29nYVi0UNDw+rp6enphjWY8dQPkXCPmc8HteBAwcUCoVcyYJMJqN4PK6jR486iaX1kh92P9r9CgBoHSQbAACAlpaWZFmWjh07JunecP/R0VFZlqUbN24428qVD42vpjQhUDr1wH5ibo9U2O75pXtJiNJExE6xaxVsFNfIyIhM09T+/ftdIzfK1dLnCwsLTnu77+xrT01NVZzLnvrQ3d0tSbp27dqW36Pt1q1bMk1Tr776qmu7aZrK5/O6ffu2xsbGZFmWrl+/7uxfXFx04rQTS+3t7erp6akYCWGPetionwAAzYlkAwAA0Pz8vCT3Df++ffskSVeuXNmRawaDQUn3bs6bRTwe37SNz+dz6hBsNEXAyz6325dPO6kl3vVcuHBBo6OjVadB+Hw+BYNBjY+Pa3p62lVE0v487c+3NLF0+fLlivOUHgMAaB0kGwAAgOspuc2+Eay2GgE2FggEdPv27YppEaW87HO7vVcrvaRSKZmm6Yym2EhfX9+m8dqJh2rvGQDQmkg2AAAApzhgtafw5QUCvbbT56+XYDCohYUFWZaliYmJiv070edeTEfIZDK6c+eOBgcHa2pfOnJBuh97tQTLekUoAQCth2QDAADQyZMnJUl37951ttk3i729vTtyTfvGuLwAYSOzkwbVbqSrMU1Ts7OzVaczeNnn09PTkqRkMumcw16dYityuZwWFxdd9S8ymYyGhobWPaZQKLjitX//3e9+52oj3X/P5cbGxrYUJwCg8ZFsAAAAOnLkiEzT1Llz55wn7Tdu3FA0GnWKDUr3n1rbiYLSgn/2DWnpE/vym117SchCoaBkMinTNF1Pu7d7/oe19OXevXud+EvZfVZtlMKJEyeq3kzX0uel57OvWXpte79dZDIej8vv98swDLW1tTk3/vaSmButTmGvaDEyMuKq/bB//34nIZRKpVyrSqyurmp5edn1Henu7tbY2JhisZgT39zcnEzTdApYlh4vSS+//PK6cQEAmhPJBgAA4BQ1NE1TbW1tToHB9957z9Xu3XfflWma6uzslGVZ6urqcp7enz17VtL95SkvXryocDjsOn7fvn0KhULy+/3q6OhQMpn09Pw77ZVXXpEkffbZZ842+8ZekqvvSo2Pj1dMIailz+3zSpLf73f9Wbo/EAgom806SY1oNKpsNuusXJHP5xWNRjdMyJw5c2bd2gudnZ2SpN27d6unp0eGYSgWi+nzzz+vOjXCfr+l76v8s5bu96PdrwCA1mEUH/YC1QCACvbTR7s6PfAgGvH7ZN9wNtp/OwzD0NWrV9XX11fzMfZoitOnT+9UWDsmFAq5ltOst1gsJr/fv+W+nJubU39/f8N9nwAAjnlGNgAAAGxBJBLRzZs3XVM8mkE6ndbo6GhnBAxtAAAXnElEQVS9w3BkMhllMhlFIpF6hwIA2AEkGwAAwI4qrTtQraZBs7GnP5w7d27DGgiNZGlpSc8++2xNS1k+DCsrK5qamtLMzIyz3CcAoLWQbAAAADuqtO5A6e/NLBAIKJlManFxsd6h1KS7u9spbtkILMvS2bNnFQgE6h0KAGCHPFHvAAAAQGtr1Xn1Pp+vKes2NAL6DQBaHyMbAAAAAACAp0g2AAAAAAAAT5FsAAAAAAAAniLZAAAAAAAAPEWBSABoEJ9++qnm5ubqHQZawKeffipJfJ9q9NFHH9U7BGwRnxkAND6j2KologGgifT29uratWv1DgMAmgr/jQWAhjVPsgEAAKzLMAxdvXpVfX199Q4FAAA0j3lqNgAAAAAAAE+RbAAAAAAAAJ4i2QAAAAAAADxFsgEAAAAAAHiKZAMAAAAAAPAUyQYAAAAAAOApkg0AAAAAAMBTJBsAAAAAAICnSDYAAAAAAABPkWwAAAAAAACeItkAAAAAAAA8RbIBAAAAAAB4imQDAAAAAADwFMkGAAAAAADgKZINAAAAAADAUyQbAAAAAACAp0g2AAAAAAAAT5FsAAAAAAAAniLZAAAAAAAAPEWyAQAAAAAAeIpkAwAAAAAA8BTJBgAAAAAA4CmSDQAAAAAAwFMkGwAAAAAAgKdINgAAAAAAAE+RbAAAAAAAAJ4i2QAAAAAAADxFsgEAAAAAAHiKZAMAAAAAAPAUyQYAAAAAAOApkg0AAAAAAMBTJBsAAAAAAICnSDYAAAAAAABPPVHvAAAAQGNIJBL67//+74rt169f17//+7+7tv385z9XIBB4WKEBAIAmYxSLxWK9gwAAAPUXjUb1L//yL3rqqafWbfPVV1/pmWee0X/913/piSd4ZgEAAKqaZxoFAACQJA0MDEiS/vd//3fdn8cff1wnT54k0QAAADZEsgEAAEiSXn31VX3729/esM1XX33lJCUAAADWQ7IBAABIkgzD0Ouvv64nn3xy3TbPPfecurq6HmJUAACgGZFsAAAAjoGBAX355ZdV9z355JN66623ZBjGQ44KAAA0G5INAADA8eKLL+qFF16ouu/LL79kCgUAAKgJyQYAAODyxhtvaNeuXRXbX3jhBf3lX/5lHSICAADNhmQDAABweeONN/T111+7tu3atUs///nP6xQRAABoNiQbAACAy5//+Z/rr/7qr1y1Gb7++mumUAAAgJqRbAAAABXefPNNPf7445LurVLx0ksv6U//9E/rHBUAAGgWJBsAAECFgYEB/eEPf5AkPf7443rzzTfrHBEAAGgmJBsAAECFb3/72/rBD34gwzD0hz/8Qb29vfUOCQAANBGSDQAAoKpwOKxisai/+Zu/0Z49e+odDgAAaCJGsVgs1jsIAHjU9fb26tq1a/UOAwCaCv+NBYCGNf9EvSMAANzT1dWld955p95hoAX84he/kCRPvk+/+MUvdOrUKe3evfuBz9WI+vv7NTw8rAMHDtQ7FGzBRx99pAsXLtQ7DADABkg2AECD+M53vqO+vr56h4EWMD8/L0mefJ9++MMf6rnnnnvg8zSq/v5+HThwgL97TYhkAwA0Nmo2AACAdbVyogEAAOwckg0AAAAAAMBTJBsAAAAAAICnSDYAAAAAAABPkWwAAAAAAACeItkAAADWFYvFFIvF6h1GQ8rlcpqcnKx3GE1pcnJShUKh3mEAAHYQyQYAANCwCoWCDMOodxgVcrmczpw5o927d8swDBmGsW5Sxt5f+tOoVldXNTQ0JMMwNDQ0pKWlpartMpmM6/0MDQ2te85MJqNEIqFQKOS898OHDyscDiuXy+3I+wAA1B/JBgAAsK7x8XGNj4/X7frLy8t1u/Z6CoWCIpGI3nrrLUWjUeXzec3Ozioej1dNOBSLRa2trUmS1tbWVCwWH3bINSkUCspkMrp06ZLy+bwOHjyonp4eWZZV0faTTz5xvT569GjVc05OTioWi2nPnj365S9/6bz3YDCo0dFRRSIRRjgAQIsi2QAAABpSoVBQIpGodxgVZmZmFAwG1dXVJUny+Xw6ceKEJCkejyuVSlUcEwgEXH82ouXlZZmmKcn9nkKhUEXbPXv2qFgsOj/2caWGhoaUz+eVTCZlmqY6Ojpc+7u6utTe3q6ZmZkdeDcAgHoj2QAAAKrK5XJKpVLOzWb5a8uyZBiGQqGQVldXnTaWZTltEomEM8x+ZWXFOXe1KQXl2yYmJpyn6qXb61lHIpfLaWRkRIcOHaq6f2JiQgMDA1UTDtUUCgWlUinn/SUSCdfUglr6vLTt5OSks3+9KRDrqZYwkKRoNOp6vbq6qlAopFgspnQ6XfUY+/MZHx+Xz+db95q9vb0aGRlhOgUAtCCSDQAAoKpIJKKBgQHnhr/0dTqdlmmaymazsixL58+flyS1tbUpFAo5bQYHB5XP5yVJnZ2dTsLBnlZQKpvNul6XTt+wn6DX28cffyxJeuGFF6ruP336tMbGxjQwMKBMJrPp+cLhsL744gtnqoVlWa6pBbX0uXQv0RCJRNTe3q5isajh4WH19PTUFMN67BjKp0jY54zH4zpw4IBCoZArWZDJZBSPx3X06FEn2bRe8sPuR7tfAQCtg2QDAACoamFhYd3X9hQCe2j81NSUJLkSAqXTDOyn43biotp0gvJh9uupZx0Ju1bBRrGOjIzINE3t37/fNZqj3NLSkizL0rFjxyTd65PR0VFZlqUbN25Iqq3PS89lT33o7u6WJF27dm3L79F269YtmaapV1991bXdNE3l83ndvn1bY2NjsixL169fd/YvLi46cdrJpvb2dvX09FSMhLBHPWzUTwCA5kSyAQAA7LhgMCjp3o14M4vH45u28fl8Th2CjaYIzM/PS3InXvbt2ydJunLlypbistuXT0WpJd71XLhwQaOjo1WnQfh8PgWDQY2Pj2t6etpVRNL+jO3PvDTZdPny5YrzlB4DAGgdJBsAAAA8FggEdPv27YppEaVKRybY7JvvaitAbMRuX1q08UGmnqRSKZmm6Yym2EhfX9+m8dqJh2rvGQDQmkg2AACAh6a82GArCwaDWlhYkGVZmpiYqNhvF2SsNvJhu/3kxXSETCajO3fuaHBwsKb2pSMXpPuxV0uwrFeEEgDQekg2AACAHWffBJcXG2w2dtKg2o10NaZpanZ2tup0hpMnT0qS7t6962yzz9vb27uluKanpyVJyWTSOYe9OsVW5HI5LS4uumpiZDIZDQ0NrXtMoVBwxWv//rvf/c7VRrr/nsuNjY1tKU4AQOMj2QAAAKoqX4Kx9LV981h6013+hN5e/rFQKCiZTMo0TdeTbfsJuJ2IKC0eaN/clj79t2+c67n05d69eyVVJhvs915tlMKJEyeq3kwfOXJEpmnq3LlzznE3btxQNBp1CjzW2ud2kcl4PC6/3y/DMNTW1ubc+NtLYm60OoW9osXIyIir9sP+/fudJFEqlXKtKrG6uqrl5WUnXuleccqxsTHFYjEnvrm5OZmm6RSwLD1ekl5++eV14wIANCeSDQAAoKq2tjbX76Wv/X6/68/y9tK9YoehUEh+v18dHR1KJpOu/e+++65M01RnZ6csy1JXV5czEuDs2bOS7i9/efHiRYXDYW/f4Da88sorkqTPPvvM2Wbf2Ev3+sAuzlhqfHy8YgqBXUjSNE3Xce+9957TptY+DwQCymazTlIjGo0qm806K1fk83lFo9ENkzRnzpxZt/ZCZ2enJGn37t3q6emRYRiKxWL6/PPPq06NsN9v6fsq//yl+/1o9ysAoHUYxUZYtBoAHnH200e7Oj3wIOr9fbJvLpvhvxiGYejq1avq6+ur+Rh7hMXp06d3KqwdEwqFKpY0radYLCa/37/lvpybm1N/f39TfMcA4BE1z8gGAACALYhEIrp586Zr2kczSKfTGh0drXcYjkwmo0wmo0gkUu9QAAA7gGQDALSQXC6nVCqlUChU71DwiCqv89CK7OkP586d27AGQiNZWlrSs88+W9NSlg/DysqKpqamNDMz4yz3CQBoLSQbAKCFnDlzRgMDA5uued+oCoWC0um0EonEhgmTTCbjKmC3UaX8akqPLf+ZnJyUZVk1rzYAt/I6D60qEAgomUxqcXGx3qHUpLu72ylu2Qgsy9LZs2cVCATqHQoAYIeQbACAFnLp0qV6h/BAJiYm9P777+vUqVMbJkw++eQT1+utLqdYLBa1trbmvM7n8yoWiyoWizp8+LASiYTC4XDLPpnfSXY/2j+tzOfzNWXdhkZw+vRpEg0A0OJINgAAGsb4+Liz+sBG9uzZ47qhrVYNfzOlNzqlw7iDwaBmZmYk3ZubzwgHAACArSPZAABNrFAoKJVKyTAMhUIhraysVG2Xy+U0OTnptFtaWnK2l9Z4sCzLabO6uuo6h318IpFQLperWN5vvWt4bXV1VaFQSLFYbN0CfbFYbMMl/jYTCAQ0PDwsy7K0vLzs2tdKfQkAALBTSDYAQBMLh8O6efOm8vm8FhYW9Otf/7qiTS6XUyQSUXt7u4rFooaHh9XT0+NUgbdrPKTTaZmmqWw2K8uydP78eecck5OT6u3tVbFYVF9fny5evFjzNbxmnzMej+vAgQMKhUI7Mt3hpZdekiR98MEHzrZW60sAAIAdUwQA1N3x48eLx48f39IxCwsLRUnF3/zmN862fD5flFQs/ed9dna2WP7PvaTi2NiY83u1/aXbJBXX1tac12tra1u6xlZVi6lUPp8v3r59uzg2NlaUVJyent6R6zRrX27n+/SoklS8evVqvcPAFl29enXDv7sAgLqbe2Ln0xkAgJ1gP3EvrTBfbQm5K1euSFLFUP14PF5TfQRJikajamtr0+zsrI4cOaJAIOAq/ufFNbbC5/MpGAwqGAyqo6NDlmVpcHDQ8+uUa6a+/PTTTzU3N1dz+0fZRx99VO8QsEV8ZgDQ+IxiscVLRQNAE+jt7ZUkzc/P13yMfTNa/s94+fb12m10nvJtKysr/9feHbu08f9xHH8d/DaHpB2SgmCXL4pQSKfWjmqhVLi4NFoL0iWFOBQKzVJJKGKwHWJb6FBRt0CNpku9wUWEujQtFHTtUIiUglk09A/wN8jdN2eiRr18k5jnA4Ixd/l8PrmL4L3v83m/FY/HnQoR6XTalYX/tD7O6iztlUol+f3+c/V9Uj92u4lEwrnIb5VjGYlE9OnTp3O/H2gV/BsLAE0rR84GAGgTxyWPrEV3d7dWV1e1tbWlWCymeDyu2dlZT/s4L5/Pp1gs5nm7P378kCT19/dXbGuFY/ngwYOKMpQ8Kh+StLy83PBx8DjbY3l5+UJ/HwCA+iPYAAAtan5+XpJOTRxo75fJZJwyjna1g1oZhqFSqaRQKKQPHz5oa2tL8Xjc0z7Oq1QqOTNDvFIsFvXu3TuZpqmBgQHn9ct+LAEAALxCsAEAWtS9e/ckHZZ5tEsrlpdInJiYkCQNDw9LOlzz7/f7ZRiGgsGgIpGIq4qDfWFr/5Tk2p5Op51+rly5onQ67Ww7qY+zKu+//LkkZbNZ12fc2dnR5uamKyAg1Vb68rh+7MoSkrS4uOh6T6sdSwAAgEYh2AAALaqrq0uFQkGdnZ26fv26JiYmdOPGDZmmqaWlJU1NTUmSAoGACoWCEomEpMMEhYVCQV1dXQoGg057fr/f9VOSa/vTp0+Vy+VkGIZyuZwrz8BJfZyFYRiu/u2LbVtHR4cGBwdlGIaSyaT29vZkmuaZ+jipH8MwtL6+rsnJSa2urioQCLje10rHEgAAoJFIEAkATeA8CSKB4/B9qp1hGFpeXtbIyEijh4IzWFlZ0ejoqPg3FgCaFgkiAQAAAACAtwg2AAAAAAAATxFsAADUlZ0L4bQHcBm1YyWR2dnZiuSuAID2Q7ABAFBXBwcHNT1weZRKpboGkOrdvleKxaJevnypjo4OJ6h2XJWUVgvAbW9vu8ZqV7+RpLt372p8fNxVgQUA0H4INgAAAE9tbm62dPteKJVKikajevz4sWKxmPb397W0tKRUKlU14HBwcKDd3V1J0u7ubtMH4L5//+76fWhoyHkeCoU0OTmpaDTKDAcAaGMEGwAAgGdKpZIWFhZatn2vLC4uKhQKqa+vT5Lk8/n08OFDSVIqlVI2m614j11q9WjJ1WZ07do118ykoyVo+/r61NnZqcXFxQaNEADQaAQbAACApMML+Ww260yNX1hYcE2FrzbF/+hr6XRalmW5thWLRVmWpXA4LElaWFhwpt7//Pnzwu1LUjKZPHaJwn+tWCwqHo+rv7+/6vZ0Oq2xsbGqAYdqTjsvxWJR2WzWOb6WZckwDIXDYe3s7FSMbXZ21tm+sbFx5s+3s7OjcDisZDKpfD5/7H6RSETxeJzlFADQpgg2AAAASdL4+Lj+/v3rTOm3LMs1Fd6e5l+uUCi4fp+ennae23e9g8GgwuGwLMtSPp/XkydPtL+/L0nq6elxAg7nbb/ZfPv2TZL0zz//VN3+/PlzJRIJjY2NaXt7+9T2Tjsv0WhUY2NjzvE1TVOFQkGWZenVq1dOO8ViUdFoVJ2dnTo4ONCzZ880ODhY0xjK2funUinduXNH4XC4akDB/vz28QAAtBeCDQAAQBsbG7IsS8PDw5IOp/JPTk7Ksiytra05rx3V1dV1atvlAYHyZQWxWEySnJkK521fOgxClAciGsnOZ3DS2OPxuEzT1M2bN12zO46q5bysrq46+9vH1+57bm6uoi17OcfAwIAk6dOnT2f6fKZpan9/X1tbW0okErIsS58/f67Yz+fzSdKJnw8AcHkRbAAAAMrlcpLcF/y9vb2SpI8fP9alz1AoJOnwwvsySaVSp+7j8/mcfAYnLTXw8rzY+x9dmlLLeI/y+XwKhUKanp7W/Py8EzA6uo90+c4vAKA2BBsAAIDrDrjNvlisdiGJiwsEAtra2qpYFlHOy/Ni7+916dmRkRG+IwCACgQbAACAU02g2h12e7lDvdS7/WYWCoW0uroqy7KUTqcrttfjvHi9rKF8SQwAADaCDQAAQI8ePZIk/fr1y3nNvtMeiUTq0qd90Ts0NFSX9hvFDhpUm6lQjWmaWlpaqrqcwcvzMj8/L0nKZDJOG3Z1iosolUonjiWRSFyofQBAayLYAAAAdP/+fZmmqZmZGecu+trammKxmJNIUPr3brodKCgvfTgxMSHJfTf+6IWsXe6xVCopk8nINE1n/4u030ylL7u7uyVVBhvs41ptlsLDhw+rXpTXcl7K27P7LO/b3m4nmUylUvL7/TIMQ8Fg0AkU2CUxT6pOkc1mXeUyd3Z2tLm56fqOlG+TpFu3bh3bHgDg8iLYAAAAnISFpmkqGAw6yQNfv37t2u/FixcyTVM9PT2yLEt9fX3OnfmpqSlJ/5anfP/+vcbHx13v7+3tVTgclt/vV1dXlzKZjKftN4Pbt29Lkv78+eO8Zl/YS3Id33LT09OuwItU23mx25Ukv9/v+lm+PRAIqFAoOEGNWCymQqHgVK7Y399XLBY7MWjT0dGhwcFBGYahZDKpvb29ijHb7M9vHw8AQHsxDpqxQDUAtBn7zqKdeR64iGb8PtkXyc32b4dhGFpeXtbIyIin7dozLp4/f+5pu/+FcDjsKqd5XslkUn6/vy7HYGVlRaOjo033fQIAOHLMbAAAAPBYNBrVly9fXMtAWkE+n9fk5OSF29ne3tb29rai0agHowIAtCKCDQAAoK7KcwpUy1dwGdnLH2ZmZk7MgdBMNjY2dPXqVfX19V2onZ8/f2pubk6Li4tOmU4AQPsh2AAAAOqqPKdA+fPLLhAIKJPJaH19vdFDqcnAwICT3PIiLMvS1NSUAoGAB6MCALSq/zV6AAAA4HJr53X1Pp+vJfM2XES7fV4AQHXMbAAAAAAAAJ4i2AAAAAAAADxFsAEAAAAAAHiKYAMAAAAAAPAUCSIBoEnk83lFIpFGDwOXQD6flyS+TzV6+/atcrlco4eBM/j9+3ejhwAAOIVx0M4pogGgSbx580Zfv35t9DAAoKUQJAKAppUj2AAAAAAAALyUI2cDAAAAAADwFMEGAAAAAADgKYINAAAAAADAUwQbAAAAAACAp/4PCKHDH7som2IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.utils import plot_model\n",
    "import pydot\n",
    "plot_model(model, to_file='model.png',show_shapes=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3b:\n",
    "transfer learning "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If you run into memory issues, run the code below before importing tensorflow (training on cpu is slower)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Even more data augmentation\n",
    "model= tf.keras.models.load_model(\"./concat_models/model0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 128, 128, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "model (Functional)              (None, 2048)         10186304    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Functional)            (None, 2048)         31957504    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 4096)         0           model[0][0]                      \n",
      "                                                                 model_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 2048)         8390656     concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 2048)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          1049088     dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          131328      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 5)            1285        dense_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 51,716,165\n",
      "Trainable params: 51,587,077\n",
      "Non-trainable params: 129,088\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "bs=256\n",
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.1,\n",
    "        zoom_range=0.1,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=10.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.1, \n",
    "        height_shift_range = 0.1)\n",
    "\n",
    "train_generator = train_datagen.flow(X_train, y, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size\n",
    "\n",
    "base_learning_rate = 0.0001\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training was manually interrupted hence the error message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "58/58 [==============================] - 2211s 38s/step - loss: 2.6109 - acc: 0.9012\n",
      "Epoch 2/2\n",
      " 5/58 [=>............................] - ETA: 34:07 - loss: 2.1752 - acc: 0.9851"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_9996/3886741090.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m history=model.fit(train_generator,\n\u001b[0;32m      4\u001b[0m                    \u001b[0msteps_per_epoch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstep_size_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m                    epochs = 2,shuffle=True)\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"./concat_models/model_aug\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#Save step 3 for potential reuse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1098\u001b[0m                 _r=1):\n\u001b[0;32m   1099\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1100\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1101\u001b[0m               \u001b[1;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1102\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    826\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    827\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 828\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    829\u001b[0m       \u001b[0mcompiler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"xla\"\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;34m\"nonXla\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    830\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    853\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    854\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 855\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    856\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    857\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2941\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m   2942\u001b[0m     return graph_function._call_flat(\n\u001b[1;32m-> 2943\u001b[1;33m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[0;32m   2944\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2945\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1917\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1918\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1919\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1920\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1921\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    558\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    559\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 560\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    561\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    562\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[1;32m---> 60\u001b[1;33m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#step3\n",
    "# start from where we ended in step 2 (same weights)\n",
    "history=model.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs = 2,shuffle=True)\n",
    "model.save(\"./concat_models/model_aug\") #Save step 3 for potential reuse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./concat_models/model_aug\\assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"./concat_models/model_aug\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 128, 128, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "model (Functional)              (None, 2048)         10186304    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Functional)            (None, 2048)         31957504    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 4096)         0           model[0][0]                      \n",
      "                                                                 model_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 2048)         8390656     concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 2048)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          1049088     dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          131328      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 5)            1285        dense_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 51,716,165\n",
      "Trainable params: 51,587,077\n",
      "Non-trainable params: 129,088\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#Even more data augmentation\n",
    "model= tf.keras.models.load_model(\"./concat_models/model_aug\")\n",
    "bs=512\n",
    "def add_noise(img):\n",
    "    '''Add random noise to an image'''\n",
    "    p=np.random.uniform()\n",
    "    if p>0.5:\n",
    "        noise = np.random.normal(0, 5, img.shape)\n",
    "        img += noise\n",
    "        np.clip(img, 0., 255.)\n",
    "    return img\n",
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.3,\n",
    "        zoom_range=0.3,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=35.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.15, \n",
    "        height_shift_range = 0.15,\n",
    "        brightness_range=(0.3, 1.4),\n",
    "        preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(X_train, y, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size\n",
    "\n",
    "base_learning_rate = 0.00005\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29/29 [==============================] - 2688s 92s/step - loss: 2.2471 - acc: 0.9441 - val_loss: 2.0447 - val_acc: 0.9915\n",
      "INFO:tensorflow:Assets written to: ./concat_models/model_final\\assets\n"
     ]
    }
   ],
   "source": [
    "#VALIDATION ACCURACY IS MEANINGLESS HERE, I forgot to remove it\n",
    "#Because we use such small number of epoch and very small learning rate, overfitting is less likely to happen\n",
    "history=model.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs = 1,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n",
    "model.save(\"./concat_models/model_final\") #Save step 3 for potential reuse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training was manually interrupted hence the error message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 128, 128, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "model (Functional)              (None, 2048)         10186304    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Functional)            (None, 2048)         31957504    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 4096)         0           model[0][0]                      \n",
      "                                                                 model_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 2048)         8390656     concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 2048)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          1049088     dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          131328      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 5)            1285        dense_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 51,716,165\n",
      "Trainable params: 51,587,077\n",
      "Non-trainable params: 129,088\n",
      "__________________________________________________________________________________________________\n",
      "14/25 [===============>..............] - ETA: 20:53 - loss: 2.1104 - acc: 0.9607"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_43180/3060432149.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     35\u001b[0m                    \u001b[0msteps_per_epoch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstep_size_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m                    \u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m                    validation_data=(Xval, yval),shuffle=True)\n\u001b[0m\u001b[0;32m     38\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"./concat_models/model_final2\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#Save step 3 for potential reuse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1098\u001b[0m                 _r=1):\n\u001b[0;32m   1099\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1100\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1101\u001b[0m               \u001b[1;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1102\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    826\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    827\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 828\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    829\u001b[0m       \u001b[0mcompiler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"xla\"\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;34m\"nonXla\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    830\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    853\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    854\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 855\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    856\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    857\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2941\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m   2942\u001b[0m     return graph_function._call_flat(\n\u001b[1;32m-> 2943\u001b[1;33m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[0;32m   2944\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2945\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1917\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1918\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1919\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1920\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1921\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    558\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    559\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 560\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    561\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    562\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32m~\\anaconda3\\envs\\envTF24\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[1;32m---> 60\u001b[1;33m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "shuffler = np.random.permutation(len(X_train))\n",
    "X_train = X_train[shuffler]\n",
    "y_train = y_train[shuffler]\n",
    "\n",
    "y=tf.keras.utils.to_categorical(y_train)\n",
    "\n",
    "bs=600\n",
    "def add_noise(img):\n",
    "    '''Add random noise to an image'''\n",
    "    p=np.random.uniform()\n",
    "    if p>0.5:\n",
    "        noise = np.random.normal(0, 5, img.shape)\n",
    "        img += noise\n",
    "        np.clip(img, 0., 255.)\n",
    "    return img\n",
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.3,\n",
    "        zoom_range=0.3,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=35.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.15, \n",
    "        height_shift_range = 0.15,\n",
    "        brightness_range=(0.3, 1.4),\n",
    "        preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(X_train, y, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size\n",
    "\n",
    "base_learning_rate = 0.00005\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "model.summary()\n",
    "model= tf.keras.models.load_model(\"./concat_models/model_final\")\n",
    "history=model.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs = 1,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n",
    "model.save(\"./concat_models/model_final2\") #Save step 3 for potential reuse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./concat_models/modelfinal2\\assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"./concat_models/modelfinal2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 128, 128, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "model (Functional)              (None, 2048)         10186304    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Functional)            (None, 2048)         31957504    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 4096)         0           model[0][0]                      \n",
      "                                                                 model_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 2048)         8390656     concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 2048)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          1049088     dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          131328      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 5)            1285        dense_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 51,716,165\n",
      "Trainable params: 51,587,077\n",
      "Non-trainable params: 129,088\n",
      "__________________________________________________________________________________________________\n",
      "21/21 [==============================] - 2775s 131s/step - loss: 2.0904 - acc: 0.9545 - val_loss: 1.9645 - val_acc: 0.9920\n"
     ]
    }
   ],
   "source": [
    "shuffler = np.random.permutation(len(X_train))\n",
    "X_train = X_train[shuffler]\n",
    "y_train = y_train[shuffler]\n",
    "\n",
    "y=tf.keras.utils.to_categorical(y_train)\n",
    "\n",
    "bs=700\n",
    "def add_noise(img):\n",
    "    '''Add random noise to an image'''\n",
    "    p=np.random.uniform()\n",
    "    if p>0.5:\n",
    "        noise = np.random.normal(0, 10, img.shape)\n",
    "        img += noise\n",
    "        np.clip(img, 0., 255.)\n",
    "    return img\n",
    "train_datagen = ImageDataGenerator(\n",
    "        shear_range=0.3,\n",
    "        zoom_range=0.2,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=38.,\n",
    "        fill_mode='wrap', \n",
    "        width_shift_range = 0.16, \n",
    "        height_shift_range = 0.16,\n",
    "        brightness_range=(0.3, 1.4),\n",
    "        preprocessing_function=add_noise)\n",
    "\n",
    "train_generator = train_datagen.flow(X_train, y, batch_size=bs)\n",
    "step_size_train=train_generator.n//train_generator.batch_size\n",
    "\n",
    "base_learning_rate = 0.00003\n",
    "model= tf.keras.models.load_model(\"./concat_models/modelfinal2\")\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss='categorical_crossentropy', metrics = ['acc'])\n",
    "model.summary()\n",
    "\n",
    "history=model.fit(train_generator,\n",
    "                   steps_per_epoch = step_size_train,\n",
    "                   epochs = 1,\n",
    "                   validation_data=(Xval, yval),shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./concat_models/modelfinal3\\assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"./concat_models/modelfinal_contact\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Lion mail drive link to the model: <a href=\"https://drive.google.com/drive/folders/13LU6Z5qObxH6pxdUo9N531-eqFGlvAIV?usp=sharing\">https://drive.google.com/drive/folders/13LU6Z5qObxH6pxdUo9N531-eqFGlvAIV?usp=sharing</a></h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating the .csv file for Kaggle\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "finalmodel = tf.keras.models.load_model(\"./concat_models/modelfinal_contact\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note that TTA does not always consistenly output the same predictions\n",
    "because different augmentation parameters are randomly chosen every time.\n",
    "\n",
    "\n",
    "Because inference performance is not a priority, tta steps can be unreasonably high, and yield marginaly better predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 20/20 [03:42<00:00, 11.13s/it]\n"
     ]
    }
   ],
   "source": [
    "#FINAL PREDICTION OBTAINED WITH TTA\n",
    "\n",
    "from tqdm import tqdm\n",
    "tta_steps = 20\n",
    "predictions = []\n",
    "bs=64\n",
    "test_datagen = ImageDataGenerator(\n",
    "        shear_range=0.1,\n",
    "        zoom_range=0.1,\n",
    "        vertical_flip=True,\n",
    "        rotation_range=10.,\n",
    "        fill_mode='reflect', \n",
    "        width_shift_range = 0.1, \n",
    "        height_shift_range = 0.1,\n",
    "        brightness_range=(0.8, 1.1)\n",
    "        )\n",
    "for i in tqdm(range(tta_steps)):\n",
    "    preds = finalmodel.predict(test_datagen.flow(X_test, batch_size=bs, shuffle=False), steps = len(X_test)/bs)\n",
    "    predictions.append(preds)\n",
    "    \n",
    "final_pred = np.mean(predictions, axis=0)\n",
    "\n",
    "predicted_values_generated_by_your_model= tf.argmax(final_pred, axis=1) # right format\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "with open('predicted2.csv','w') as csvfile:\n",
    "    fieldnames = ['Id','label']\n",
    "    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "    writer.writeheader()\n",
    "    for index,l in enumerate(predicted_values_generated_by_your_model.numpy()):\n",
    "        filename = str(index) + '.png'\n",
    "        label = str(l)\n",
    "        writer.writerow({'Id': filename, 'label': label})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "tree is already the newest version (1.7.0-5).\n",
      "The following packages were automatically installed and are no longer required:\n",
      "  libnuma1 linux-gcp-5.4-headers-5.4.0-1052\n",
      "Use 'sudo apt autoremove' to remove them.\n",
      "0 upgraded, 0 newly installed, 0 to remove and 51 not upgraded.\n"
     ]
    }
   ],
   "source": [
    "!sudo apt-get install tree\n",
    "!tree ./ >> README.md"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}