caro_bot_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from abc import ABC, abstractmethod\n",
    "from tensorflow.keras import Sequential\n",
    "import numpy as np\n",
    "import random\n",
    "import pickle\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras import losses\n",
    "from tensorflow.keras import optimizers\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaseModel(ABC):\n",
    "    @abstractmethod\n",
    "    def __init__(self, discount_factor: float, epsilon: float, e_min: int, e_max: int):\n",
    "        \"\"\"\n",
    "        self.qNetwork and self.qTargetNetwork need to be installed.\n",
    "\n",
    "        :param discount_factor: It’s used to balance immediate and future reward.\n",
    "        Typically this value can range anywhere from 0.8 to 0.99.\n",
    "\n",
    "        :param epsilon: the probability of choosing to explore action.\n",
    "        :param e_min: Minimum amount of experience to start training.\n",
    "        :param e_max: Maximum amount of experience.\n",
    "        \"\"\"\n",
    "        pass\n",
    "\n",
    "    @abstractmethod\n",
    "    def observe(self, state, action_space: list = None):\n",
    "        \"\"\"\n",
    "        Observe state from environment and return a action\n",
    "\n",
    "        :param state: Current situation returned by the environment.\n",
    "        :param action_space: All the possible moves that the agent can take.\n",
    "        :return: Action that have the max value from q table value.\n",
    "\n",
    "        Note: If actionSpace is None then all action are possible\n",
    "        \"\"\"\n",
    "        pass\n",
    "\n",
    "    @abstractmethod\n",
    "    def observe_on_training(self, state, action_space: list = None) -> int:\n",
    "        \"\"\"\n",
    "        Observe state from environment and return a action by epsilon greedy policy.\n",
    "        The state and action will be stored in the memory buffer to be used for Experience Replay\n",
    "\n",
    "        :param state: Current situation returned by the environment.\n",
    "        :param action_space: All the possible moves that the agent can take.\n",
    "        :return: Action that have the max value from q table value.\n",
    "        Note: If actionSpace is None then all action are possible\n",
    "        \"\"\"\n",
    "        pass\n",
    "\n",
    "    @abstractmethod\n",
    "    def take_reward(self, reward, next_state, done):\n",
    "        \"\"\"\n",
    "        After used observeOnTraining method, environment will return reward, nextState and done information\n",
    "        we will use this method to put that information into the Experience Replay\n",
    "\n",
    "        :param reward: immediate reward returned by the environment\n",
    "        :param next_state: Next situation returned by the environment.\n",
    "        :param done: describes whether the environment situation has terminated or not\n",
    "        :return: None\n",
    "        \"\"\"\n",
    "        pass\n",
    "\n",
    "    @abstractmethod\n",
    "    def train_network(self, sample_size: int, batch_size: int, epochs: int, verbose: int = 2, cer_mode: bool = False):\n",
    "        \"\"\"\n",
    "        :param sample_size: number of samples taken from Experience Replay.\n",
    "        :param batch_size: Integer or `None`. Number of samples per gradient update. If unspecified,\n",
    "        `batch_size` will default to 32. Do not specify the `batch_size` if your data is in the form of datasets,\n",
    "        generators, or `keras.utils.Sequence` instances (since they generate batches).\n",
    "        :param epochs: Integer. Number of epochs to train the model.\n",
    "        An epoch is an iteration over the entire `x` and `y` data provided.\n",
    "        Note that in conjunction with `initial_epoch`, `epochs` is to be understood as \"final epoch\".\n",
    "        The model is not trained for a number of iterations given by `epochs`, but merely until the epoch of index\n",
    "        `epochs` is reached.\n",
    "        :param verbose: 0, 1, or 2. Verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch.\n",
    "        Note that the progress bar is not particularly useful when logged to a file, so verbose=2 is recommended\n",
    "        when not running interactively (eg, in a production environment).\n",
    "        :param cer_mode: Turn on or off cer (Combined Experience Replay). Default is False.\n",
    "        :return: None\n",
    "        \"\"\"\n",
    "        pass\n",
    "\n",
    "    @abstractmethod\n",
    "    def update_target_network(self):\n",
    "        \"\"\"\n",
    "        Update Q target Network by weight of Q network\n",
    "        :return: None\n",
    "        \"\"\"\n",
    "        pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EpsilonGreedy:\n",
    "    def __init__(self, epsilon):\n",
    "        self.epsilon = epsilon\n",
    "\n",
    "    def perform(self, q_value, action_space: list = None):\n",
    "        prob = np.random.sample()  # get probability of taking random action\n",
    "        if prob <= self.epsilon:  # take random action\n",
    "            if action_space is None:  # all action are available\n",
    "                return np.random.randint(len(q_value))\n",
    "            return np.random.choice(action_space)\n",
    "        else:  # take greedy action\n",
    "            if action_space is None:\n",
    "                return np.argmax(q_value)\n",
    "            return max([[q_value[a], a] for a in action_space], key=lambda x: x[0])[1]\n",
    "\n",
    "    def decay(self, decay_value, lower_bound):\n",
    "        \"\"\"\n",
    "        Adjust the epsilon value by the formula: epsilon = max(decayValue * epsilon, lowerBound).\n",
    "        :param decay_value: Value ratio adjustment (0, 1).\n",
    "        :param lower_bound: Minimum epsilon value.\n",
    "        :return: None\n",
    "        \"\"\"\n",
    "        self.epsilon = max(self.epsilon * decay_value, lower_bound)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ExperienceReplay:\n",
    "    def __init__(self, e_max: int):\n",
    "        if e_max <= 0:\n",
    "            raise ValueError('Invalid value for memory size')\n",
    "        self.e_max = e_max\n",
    "        self.memory = list()\n",
    "        self.index = 0\n",
    "\n",
    "    def add_experience(self, sample: list):\n",
    "        if len(sample) != 5:\n",
    "            raise Exception('Invalid sample')\n",
    "        if len(self.memory) < self.e_max:\n",
    "            self.memory.append(sample)\n",
    "        else:\n",
    "            self.memory[self.index] = sample\n",
    "        self.index = (self.index + 1) % self.e_max\n",
    "\n",
    "    def sample_experience(self, sample_size: int, cer_mode: bool):\n",
    "        samples = random.sample(self.memory, sample_size)\n",
    "        if cer_mode:\n",
    "            samples[-1] = self.memory[self.index - 1]\n",
    "        # state_samples, action_samples, reward_samples, next_state_samples, done_samples\n",
    "        s_batch, a_batch, r_batch, ns_batch, done_batch = map(np.array, zip(*samples))\n",
    "        return s_batch, a_batch, r_batch, ns_batch, done_batch\n",
    "\n",
    "    def get_size(self):\n",
    "        return len(self.memory)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(BaseModel):\n",
    "    def __init__(self, discount_factor: float, epsilon: float, e_min: int, e_max: int):\n",
    "        super().__init__(discount_factor, epsilon, e_min, e_max)\n",
    "        self.gamma = discount_factor\n",
    "        self.epsilon_greedy = EpsilonGreedy(epsilon)\n",
    "        self.e_min = e_min\n",
    "        self.exp_replay = ExperienceReplay(e_max)\n",
    "        self.training_network = Sequential()\n",
    "        self.target_network = Sequential()\n",
    "        self.cache = list()\n",
    "\n",
    "    def observe(self, state, action_space: list = None):\n",
    "        q_value = self.training_network.predict(np.array([state])).ravel()\n",
    "        if action_space is not None:\n",
    "            return max([[q_value[a], a] for a in action_space], key=lambda x: x[0])[1]\n",
    "        return np.argmax(q_value)\n",
    "\n",
    "    def observe_on_training(self, state, action_space: list = None) -> int:\n",
    "        q_value = self.training_network.predict(np.array([state])).ravel()\n",
    "        action = self.epsilon_greedy.perform(q_value, action_space)\n",
    "        self.cache.extend([state, action])\n",
    "        return action\n",
    "\n",
    "    def take_reward(self, reward, next_state, done):\n",
    "        self.cache.extend([reward, next_state, done])\n",
    "        self.exp_replay.add_experience(self.cache.copy())\n",
    "        self.cache.clear()\n",
    "\n",
    "    def train_network(self, sample_size: int, batch_size: int, epochs: int, verbose: int = 2, cer_mode: bool = False):\n",
    "        if self.exp_replay.get_size() >= self.e_min:\n",
    "            # state_samples, action_samples, reward_samples, next_state_samples, done_samples\n",
    "            s_batch, a_batch, r_batch, ns_batch, done_batch = self.exp_replay.sample_experience(sample_size, cer_mode)\n",
    "            states, q_values = self.replay(s_batch, a_batch, r_batch, ns_batch, done_batch)\n",
    "            history = self.training_network.fit(states, q_values, epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
    "            return history.history['loss']\n",
    "\n",
    "    def replay(self, states, actions, rewards, next_states, terminals):\n",
    "        q_values = self.target_network.predict(np.array(states))  # get q value at state t by target network\n",
    "        nq_values = self.target_network.predict(np.array(next_states))  # get q value at state t+1 by target network\n",
    "        for i in range(len(states)):\n",
    "            a = actions[i]\n",
    "            done = terminals[i]\n",
    "            r = rewards[i]\n",
    "            if done:\n",
    "                q_values[i][a] = r\n",
    "            else:\n",
    "                q_values[i][a] = r + self.gamma * np.max(nq_values[i])\n",
    "        return states, q_values\n",
    "\n",
    "    def update_target_network(self):\n",
    "        self.target_network.set_weights(self.training_network.get_weights())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tictactoe_v0:\n",
    "    def __init__(self):\n",
    "        self.board = [0] * 9\n",
    "        self.wining_position = [[0, 1, 2], [3, 4, 5], [6, 7, 8],\n",
    "                                [0, 3, 6], [1, 4, 7], [2, 5, 8],\n",
    "                                [0, 4, 8], [6, 4, 2]]\n",
    "        self.current_turn = 1\n",
    "        self.player_mark = 1\n",
    "\n",
    "    def reset(self, is_human_first):\n",
    "        self.board = [0] * 9\n",
    "        self.current_turn = 1\n",
    "        self.player_mark = 1 if is_human_first else -1\n",
    "        if not is_human_first:\n",
    "            self.env_act()\n",
    "        return self.board.copy()\n",
    "\n",
    "    def check_win(self):\n",
    "        for pst in self.wining_position:\n",
    "            if str(self.board[pst[0]]) + str(self.board[pst[1]]) + str(self.board[pst[2]]) in ['111', '-1-1-1']:\n",
    "                if self.current_turn == self.player_mark:\n",
    "                    return 1, True\n",
    "                return -1, True\n",
    "        if 0 not in self.board:\n",
    "            return 0, True\n",
    "        return 0, False\n",
    "\n",
    "    def env_act(self):\n",
    "        action = random.choice([i for i in range(len(self.board)) if self.board[i] == 0])\n",
    "        for pst in self.wining_position:\n",
    "            com = str(self.board[pst[0]]) + str(self.board[pst[1]]) + str(self.board[pst[2]])\n",
    "            if com.replace('0', '') == str(self.current_turn) * 2:\n",
    "                if self.board[pst[0]] == 0:\n",
    "                    action = pst[0]\n",
    "                elif self.board[pst[1]] == 0:\n",
    "                    action = pst[1]\n",
    "                else:\n",
    "                    action = pst[2]\n",
    "        if self.board[action] != 0:\n",
    "            raise Exception('Invalid action')\n",
    "        self.board[action] = self.current_turn\n",
    "        reward, done = self.check_win()\n",
    "        self.current_turn = self.current_turn * -1\n",
    "        return reward, done\n",
    "\n",
    "    def step(self, action):\n",
    "        if self.board[action] != 0:\n",
    "            raise Exception('Invalid action')\n",
    "        self.board[action] = self.current_turn\n",
    "        reward, done = self.check_win()\n",
    "        self.current_turn = self.current_turn * -1\n",
    "        if done:\n",
    "            return self.board.copy(), reward, done, None\n",
    "        reward, done = self.env_act()\n",
    "        return self.board.copy(), reward, done, None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "env = Tictactoe_v0()\n",
    "agent = DQN(0.7, 1, 4096, 1048576)\n",
    "op1 = optimizers.RMSprop(learning_rate=0.00025)\n",
    "agent.training_network.add(Dense(128, activation='relu', input_shape=(9,)))\n",
    "agent.training_network.add(Dense(128, activation='relu'))\n",
    "agent.training_network.add(Dense(9, activation='linear'))\n",
    "agent.training_network.compile(optimizer=op1, loss=losses.mean_squared_error)\n",
    "\n",
    "\n",
    "op2 = optimizers.RMSprop(learning_rate=0.00025)\n",
    "agent.target_network.add(Dense(128, activation='relu', input_shape=(9,)))\n",
    "agent.target_network.add(Dense(128, activation='relu'))\n",
    "agent.target_network.add(Dense(9, activation='linear'))\n",
    "agent.target_network.compile(optimizer=op2, loss=losses.mean_squared_error)\n",
    "agent.update_target_network()\n",
    "reward_records = list()\n",
    "loss_records = list()\n",
    "count = 0\n",
    "tau = 500\n",
    "record = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    for ep in range(5000):\n",
    "        state = env.reset(1)\n",
    "        done = False\n",
    "        print(ep, '------------------', 'current epsilon: ', agent.epsilon_greedy.epsilon)\n",
    "        while not done:\n",
    "            action = agent.observe_on_training(state, [i for i in range(len(state)) if state[i] == 0])\n",
    "            state, reward, done, _ = env.step(action)\n",
    "            print(state, done)\n",
    "            record += reward\n",
    "            print(ep, '-----------------------------------', reward)\n",
    "            agent.take_reward(reward, state, done)\n",
    "            hist = agent.train_network(64 ,64, 1, 2, cer_mode=True)\n",
    "            loss_records.append(hist)\n",
    "            count += 1\n",
    "            if count % tau == 0:\n",
    "                agent.update_target_network()\n",
    "        reward_records.append(record)\n",
    "        agent.epsilon_greedy.decay(0.99999, 0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Plot total rewards through training process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOpUlEQVR4nO3cf6jd9X3H8eeruTRrEUyi8UeN2bVVGHGDFg5K2QauaoyDNtL6h90fDVtL/lj9Y5VCUxzT2v6hbp2ltNsIbSEIa3SO0kApEm2FMYb1xDrarE1zjS0mVZuaIDipkvW9P+7X7Xg5Mffec+49OX6eDzjc8/1+P/fe98cLeeac742pKiRJ7XrbpAeQJE2WIZCkxhkCSWqcIZCkxhkCSWrczKQHWI7zzz+/ZmdnJz2GJE2VAwcO/LqqNi48P5UhmJ2dpd/vT3oMSZoqSX4x7LxvDUlS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS48YSgiTbkhxKMpdk15Dra5M80F1/PMnsguubk7yc5NPjmEeStHgjhyDJGuCrwI3AFuCjSbYsWPZx4GRVXQ7cB9yz4PrfA98ddRZJ0tKN4xXBVcBcVR2pqteAvcD2BWu2A3u65w8B1yYJQJKbgGeAg2OYRZK0ROMIwSXAswPHR7tzQ9dU1SngJeC8JOcAnwE+d6ZvkmRnkn6S/vHjx8cwtiQJJn+z+E7gvqp6+UwLq2p3VfWqqrdx48aVn0ySGjEzhq9xDLh04HhTd27YmqNJZoBzgReBq4Gbk9wLrAN+m+Q3VfWVMcwlSVqEcYTgCeCKJJcx/wf+LcCfLVizD9gB/AdwM/C9qirgj19fkORO4GUjIEmra+QQVNWpJLcCDwNrgG9U1cEkdwH9qtoHfB24P8kccIL5WEiSzgKZ/4v5dOn1etXv9yc9hiRNlSQHqqq38PykbxZLkibMEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS48YSgiTbkhxKMpdk15Dra5M80F1/PMlsd/76JAeS/Kj7+IFxzCNJWryRQ5BkDfBV4EZgC/DRJFsWLPs4cLKqLgfuA+7pzv8a+GBV/QGwA7h/1HkkSUszjlcEVwFzVXWkql4D9gLbF6zZDuzpnj8EXJskVfXDqvpld/4g8I4ka8cwkyRpkcYRgkuAZweOj3bnhq6pqlPAS8B5C9Z8BHiyql4dw0ySpEWamfQAAEmuZP7toq1vsmYnsBNg8+bNqzSZJL31jeMVwTHg0oHjTd25oWuSzADnAi92x5uAbwEfq6qnT/dNqmp3VfWqqrdx48YxjC1JgvGE4AngiiSXJXk7cAuwb8GafczfDAa4GfheVVWSdcB3gF1V9e9jmEWStEQjh6B7z/9W4GHgJ8CDVXUwyV1JPtQt+zpwXpI54Dbg9V8xvRW4HPibJE91jwtGnUmStHipqknPsGS9Xq/6/f6kx5CkqZLkQFX1Fp73XxZLUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuPGEoIk25IcSjKXZNeQ62uTPNBdfzzJ7MC1z3bnDyW5YRzzSJIWb+QQJFkDfBW4EdgCfDTJlgXLPg6crKrLgfuAe7rP3QLcAlwJbAP+oft6kqRVMo5XBFcBc1V1pKpeA/YC2xes2Q7s6Z4/BFybJN35vVX1alU9A8x1X0+StErGEYJLgGcHjo9254auqapTwEvAeYv8XACS7EzST9I/fvz4GMaWJMEU3Syuqt1V1auq3saNGyc9jiS9ZYwjBMeASweON3Xnhq5JMgOcC7y4yM+VJK2gcYTgCeCKJJcleTvzN3/3LVizD9jRPb8Z+F5VVXf+lu63ii4DrgB+MIaZJEmLNDPqF6iqU0luBR4G1gDfqKqDSe4C+lW1D/g6cH+SOeAE87GgW/cg8F/AKeCTVfU/o84kSVq8zP/FfLr0er3q9/uTHkOSpkqSA1XVW3h+am4WS5JWhiGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMaNFIIkG5LsT3K4+7j+NOt2dGsOJ9nRnXtnku8k+WmSg0nuHmUWSdLyjPqKYBfwaFVdATzaHb9Bkg3AHcDVwFXAHQPB+Luq+j3gfcAfJrlxxHkkSUs0agi2A3u653uAm4asuQHYX1UnquoksB/YVlWvVNX3AarqNeBJYNOI80iSlmjUEFxYVc91z58HLhyy5hLg2YHjo925/5NkHfBB5l9VSJJW0cyZFiR5BLhoyKXbBw+qqpLUUgdIMgN8E/hyVR15k3U7gZ0AmzdvXuq3kSSdxhlDUFXXne5akheSXFxVzyW5GPjVkGXHgGsGjjcBjw0c7wYOV9WXzjDH7m4tvV5vycGRJA036ltD+4Ad3fMdwLeHrHkY2JpkfXeTeGt3jiRfAM4F/mrEOSRJyzRqCO4Grk9yGLiuOyZJL8nXAKrqBPB54InucVdVnUiyifm3l7YATyZ5KsknRpxHkrREqZq+d1l6vV71+/1JjyFJUyXJgarqLTzvvyyWpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMaNFIIkG5LsT3K4+7j+NOt2dGsOJ9kx5Pq+JD8eZRZJ0vKM+opgF/BoVV0BPNodv0GSDcAdwNXAVcAdg8FI8mHg5RHnkCQt06gh2A7s6Z7vAW4asuYGYH9Vnaiqk8B+YBtAknOA24AvjDiHJGmZRg3BhVX1XPf8eeDCIWsuAZ4dOD7anQP4PPBF4JUzfaMkO5P0k/SPHz8+wsiSpEEzZ1qQ5BHgoiGXbh88qKpKUov9xkneC7ynqj6VZPZM66tqN7AboNfrLfr7SJLe3BlDUFXXne5akheSXFxVzyW5GPjVkGXHgGsGjjcBjwHvB3pJft7NcUGSx6rqGiRJq2bUt4b2Aa//FtAO4NtD1jwMbE2yvrtJvBV4uKr+sareVVWzwB8BPzMCkrT6Rg3B3cD1SQ4D13XHJOkl+RpAVZ1g/l7AE93jru6cJOkskKrpe7u91+tVv9+f9BiSNFWSHKiq3sLz/stiSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxqWqJj3DkiU5Dvxi0nMs0fnAryc9xCpzz21wz9Pjd6tq48KTUxmCaZSkX1W9Sc+xmtxzG9zz9POtIUlqnCGQpMYZgtWze9IDTIB7boN7nnLeI5CkxvmKQJIaZwgkqXGGYIySbEiyP8nh7uP606zb0a05nGTHkOv7kvx45Sce3Sh7TvLOJN9J8tMkB5PcvbrTL02SbUkOJZlLsmvI9bVJHuiuP55kduDaZ7vzh5LcsKqDj2C5e05yfZIDSX7UffzAqg+/DKP8jLvrm5O8nOTTqzb0OFSVjzE9gHuBXd3zXcA9Q9ZsAI50H9d3z9cPXP8w8M/Ajye9n5XeM/BO4E+6NW8H/g24cdJ7Os0+1wBPA+/uZv1PYMuCNX8J/FP3/Bbgge75lm79WuCy7uusmfSeVnjP7wPe1T3/feDYpPezkvsduP4Q8C/Apye9n6U8fEUwXtuBPd3zPcBNQ9bcAOyvqhNVdRLYD2wDSHIOcBvwhZUfdWyWveeqeqWqvg9QVa8BTwKbVn7kZbkKmKuqI92se5nf+6DB/xYPAdcmSXd+b1W9WlXPAHPd1zvbLXvPVfXDqvpld/4g8I4ka1dl6uUb5WdMkpuAZ5jf71QxBON1YVU91z1/HrhwyJpLgGcHjo925wA+D3wReGXFJhy/UfcMQJJ1wAeBR1dgxnE44x4G11TVKeAl4LxFfu7ZaJQ9D/oI8GRVvbpCc47Lsvfb/SXuM8DnVmHOsZuZ9ADTJskjwEVDLt0+eFBVlWTRv5ub5L3Ae6rqUwvfd5y0ldrzwNefAb4JfLmqjixvSp2NklwJ3ANsnfQsK+xO4L6qerl7gTBVDMESVdV1p7uW5IUkF1fVc0kuBn41ZNkx4JqB403AY8D7gV6SnzP/c7kgyWNVdQ0TtoJ7ft1u4HBVfWn0aVfMMeDSgeNN3blha452cTsXeHGRn3s2GmXPJNkEfAv4WFU9vfLjjmyU/V4N3JzkXmAd8Nskv6mqr6z41OMw6ZsUb6UH8Le88cbpvUPWbGD+fcT13eMZYMOCNbNMz83ikfbM/P2QfwXeNum9nGGfM8zf5L6M/7+ReOWCNZ/kjTcSH+yeX8kbbxYfYTpuFo+y53Xd+g9Peh+rsd8Fa+5kym4WT3yAt9KD+fdGHwUOA48M/GHXA742sO4vmL9hOAf8+ZCvM00hWPaemf8bVwE/AZ7qHp+Y9J7eZK9/CvyM+d8sub07dxfwoe757zD/GyNzwA+Adw987u3d5x3iLP3NqHHuGfhr4L8Hfq5PARdMej8r+TMe+BpTFwL/FxOS1Dh/a0iSGmcIJKlxhkCSGmcIJKlxhkCSGmcIJKlxhkCSGve/5wv9yACcdLkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(len(reward_records)),  reward_records)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Plot loss through training process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOpUlEQVR4nO3cf6jd9X3H8eeruTRrEUyi8UeN2bVVGHGDFg5K2QauaoyDNtL6h90fDVtL/lj9Y5VCUxzT2v6hbp2ltNsIbSEIa3SO0kApEm2FMYb1xDrarE1zjS0mVZuaIDipkvW9P+7X7Xg5Mffec+49OX6eDzjc8/1+P/fe98cLeeac742pKiRJ7XrbpAeQJE2WIZCkxhkCSWqcIZCkxhkCSWrczKQHWI7zzz+/ZmdnJz2GJE2VAwcO/LqqNi48P5UhmJ2dpd/vT3oMSZoqSX4x7LxvDUlS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS48YSgiTbkhxKMpdk15Dra5M80F1/PMnsguubk7yc5NPjmEeStHgjhyDJGuCrwI3AFuCjSbYsWPZx4GRVXQ7cB9yz4PrfA98ddRZJ0tKN4xXBVcBcVR2pqteAvcD2BWu2A3u65w8B1yYJQJKbgGeAg2OYRZK0ROMIwSXAswPHR7tzQ9dU1SngJeC8JOcAnwE+d6ZvkmRnkn6S/vHjx8cwtiQJJn+z+E7gvqp6+UwLq2p3VfWqqrdx48aVn0ySGjEzhq9xDLh04HhTd27YmqNJZoBzgReBq4Gbk9wLrAN+m+Q3VfWVMcwlSVqEcYTgCeCKJJcx/wf+LcCfLVizD9gB/AdwM/C9qirgj19fkORO4GUjIEmra+QQVNWpJLcCDwNrgG9U1cEkdwH9qtoHfB24P8kccIL5WEiSzgKZ/4v5dOn1etXv9yc9hiRNlSQHqqq38PykbxZLkibMEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS4wyBJDXOEEhS48YSgiTbkhxKMpdk15Dra5M80F1/PMlsd/76JAeS/Kj7+IFxzCNJWryRQ5BkDfBV4EZgC/DRJFsWLPs4cLKqLgfuA+7pzv8a+GBV/QGwA7h/1HkkSUszjlcEVwFzVXWkql4D9gLbF6zZDuzpnj8EXJskVfXDqvpld/4g8I4ka8cwkyRpkcYRgkuAZweOj3bnhq6pqlPAS8B5C9Z8BHiyql4dw0ySpEWamfQAAEmuZP7toq1vsmYnsBNg8+bNqzSZJL31jeMVwTHg0oHjTd25oWuSzADnAi92x5uAbwEfq6qnT/dNqmp3VfWqqrdx48YxjC1JgvGE4AngiiSXJXk7cAuwb8GafczfDAa4GfheVVWSdcB3gF1V9e9jmEWStEQjh6B7z/9W4GHgJ8CDVXUwyV1JPtQt+zpwXpI54Dbg9V8xvRW4HPibJE91jwtGnUmStHipqknPsGS9Xq/6/f6kx5CkqZLkQFX1Fp73XxZLUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuMMgSQ1zhBIUuPGEoIk25IcSjKXZNeQ62uTPNBdfzzJ7MC1z3bnDyW5YRzzSJIWb+QQJFkDfBW4EdgCfDTJlgXLPg6crKrLgfuAe7rP3QLcAlwJbAP+oft6kqRVMo5XBFcBc1V1pKpeA/YC2xes2Q7s6Z4/BFybJN35vVX1alU9A8x1X0+StErGEYJLgGcHjo9254auqapTwEvAeYv8XACS7EzST9I/fvz4GMaWJMEU3Syuqt1V1auq3saNGyc9jiS9ZYwjBMeASweON3Xnhq5JMgOcC7y4yM+VJK2gcYTgCeCKJJcleTvzN3/3LVizD9jRPb8Z+F5VVXf+lu63ii4DrgB+MIaZJEmLNDPqF6iqU0luBR4G1gDfqKqDSe4C+lW1D/g6cH+SOeAE87GgW/cg8F/AKeCTVfU/o84kSVq8zP/FfLr0er3q9/uTHkOSpkqSA1XVW3h+am4WS5JWhiGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMaNFIIkG5LsT3K4+7j+NOt2dGsOJ9nRnXtnku8k+WmSg0nuHmUWSdLyjPqKYBfwaFVdATzaHb9Bkg3AHcDVwFXAHQPB+Luq+j3gfcAfJrlxxHkkSUs0agi2A3u653uAm4asuQHYX1UnquoksB/YVlWvVNX3AarqNeBJYNOI80iSlmjUEFxYVc91z58HLhyy5hLg2YHjo925/5NkHfBB5l9VSJJW0cyZFiR5BLhoyKXbBw+qqpLUUgdIMgN8E/hyVR15k3U7gZ0AmzdvXuq3kSSdxhlDUFXXne5akheSXFxVzyW5GPjVkGXHgGsGjjcBjw0c7wYOV9WXzjDH7m4tvV5vycGRJA036ltD+4Ad3fMdwLeHrHkY2JpkfXeTeGt3jiRfAM4F/mrEOSRJyzRqCO4Grk9yGLiuOyZJL8nXAKrqBPB54InucVdVnUiyifm3l7YATyZ5KsknRpxHkrREqZq+d1l6vV71+/1JjyFJUyXJgarqLTzvvyyWpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMYZAklqnCGQpMaNFIIkG5LsT3K4+7j+NOt2dGsOJ9kx5Pq+JD8eZRZJ0vKM+opgF/BoVV0BPNodv0GSDcAdwNXAVcAdg8FI8mHg5RHnkCQt06gh2A7s6Z7vAW4asuYGYH9Vnaiqk8B+YBtAknOA24AvjDiHJGmZRg3BhVX1XPf8eeDCIWsuAZ4dOD7anQP4PPBF4JUzfaMkO5P0k/SPHz8+wsiSpEEzZ1qQ5BHgoiGXbh88qKpKUov9xkneC7ynqj6VZPZM66tqN7AboNfrLfr7SJLe3BlDUFXXne5akheSXFxVzyW5GPjVkGXHgGsGjjcBjwHvB3pJft7NcUGSx6rqGiRJq2bUt4b2Aa//FtAO4NtD1jwMbE2yvrtJvBV4uKr+sareVVWzwB8BPzMCkrT6Rg3B3cD1SQ4D13XHJOkl+RpAVZ1g/l7AE93jru6cJOkskKrpe7u91+tVv9+f9BiSNFWSHKiq3sLz/stiSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxhkCSWqcIZCkxqWqJj3DkiU5Dvxi0nMs0fnAryc9xCpzz21wz9Pjd6tq48KTUxmCaZSkX1W9Sc+xmtxzG9zz9POtIUlqnCGQpMYZgtWze9IDTIB7boN7nnLeI5CkxvmKQJIaZwgkqXGGYIySbEiyP8nh7uP606zb0a05nGTHkOv7kvx45Sce3Sh7TvLOJN9J8tMkB5PcvbrTL02SbUkOJZlLsmvI9bVJHuiuP55kduDaZ7vzh5LcsKqDj2C5e05yfZIDSX7UffzAqg+/DKP8jLvrm5O8nOTTqzb0OFSVjzE9gHuBXd3zXcA9Q9ZsAI50H9d3z9cPXP8w8M/Ajye9n5XeM/BO4E+6NW8H/g24cdJ7Os0+1wBPA+/uZv1PYMuCNX8J/FP3/Bbgge75lm79WuCy7uusmfSeVnjP7wPe1T3/feDYpPezkvsduP4Q8C/Apye9n6U8fEUwXtuBPd3zPcBNQ9bcAOyvqhNVdRLYD2wDSHIOcBvwhZUfdWyWveeqeqWqvg9QVa8BTwKbVn7kZbkKmKuqI92se5nf+6DB/xYPAdcmSXd+b1W9WlXPAHPd1zvbLXvPVfXDqvpld/4g8I4ka1dl6uUb5WdMkpuAZ5jf71QxBON1YVU91z1/HrhwyJpLgGcHjo925wA+D3wReGXFJhy/UfcMQJJ1wAeBR1dgxnE44x4G11TVKeAl4LxFfu7ZaJQ9D/oI8GRVvbpCc47Lsvfb/SXuM8DnVmHOsZuZ9ADTJskjwEVDLt0+eFBVlWTRv5ub5L3Ae6rqUwvfd5y0ldrzwNefAb4JfLmqjixvSp2NklwJ3ANsnfQsK+xO4L6qerl7gTBVDMESVdV1p7uW5IUkF1fVc0kuBn41ZNkx4JqB403AY8D7gV6SnzP/c7kgyWNVdQ0TtoJ7ft1u4HBVfWn0aVfMMeDSgeNN3blha452cTsXeHGRn3s2GmXPJNkEfAv4WFU9vfLjjmyU/V4N3JzkXmAd8Nskv6mqr6z41OMw6ZsUb6UH8Le88cbpvUPWbGD+fcT13eMZYMOCNbNMz83ikfbM/P2QfwXeNum9nGGfM8zf5L6M/7+ReOWCNZ/kjTcSH+yeX8kbbxYfYTpuFo+y53Xd+g9Peh+rsd8Fa+5kym4WT3yAt9KD+fdGHwUOA48M/GHXA742sO4vmL9hOAf8+ZCvM00hWPaemf8bVwE/AZ7qHp+Y9J7eZK9/CvyM+d8sub07dxfwoe757zD/GyNzwA+Adw987u3d5x3iLP3NqHHuGfhr4L8Hfq5PARdMej8r+TMe+BpTFwL/FxOS1Dh/a0iSGmcIJKlxhkCSGmcIJKlxhkCSGmcIJKlxhkCSGve/5wv9yACcdLkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "loss = [(sum(loss)/len(loss))for loss in loss_records if loss != None]\n",
    "plt.plot(range(len(loss)),  loss)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### After training we will save the results and start the evaluation process\n",
    "To conduct the evaluation process I will divide it into 2 parts\n",
    "* play agent againt Minimax algorithm which is alway win or draw\n",
    "* play agent againt random player which is alway lose or draw\n",
    "\n",
    "Each type of rating will undergo 1000 matches and the first move of each game will be random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('save/tictactoe_save_27-8.pkl', 'wb') as f:\n",
    "    pickle.dump(agent.training_network.get_weights(), f,pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "class Tictactoe_test_vs_minimax:\n",
    "    def __init__(self):\n",
    "        self.board = [0] * 9\n",
    "        self.wining_position = [[0, 1, 2], [3, 4, 5], [6, 7, 8],\n",
    "                                [0, 3, 6], [1, 4, 7], [2, 5, 8],\n",
    "                                [0, 4, 8], [6, 4, 2]]\n",
    "        self.current_turn = 1\n",
    "        self.player_mark = 1\n",
    "        \n",
    "    def checkWin(self, currentTable):\n",
    "        for a1, a2, a3 in self.wining_position:\n",
    "            combo = str(currentTable[a1]) + str(currentTable[a2]) + str(currentTable[a3])\n",
    "            if combo == '111':\n",
    "                return 1\n",
    "            if combo == '-1-1-1':\n",
    "                return -1\n",
    "        if 0 not in currentTable:\n",
    "            return 0.1\n",
    "        return 0\n",
    "    \n",
    "    def minimax(self, state, mark, depth, node, maximizingPlayer, alpha, beta):\n",
    "        r = self.checkWin(state)\n",
    "        if r:\n",
    "            return [r, node]\n",
    "        elif depth == 0:\n",
    "            return [0, node]\n",
    "\n",
    "        if maximizingPlayer:\n",
    "            maxScore = [float('-inf'), node]\n",
    "            for i in range(9):\n",
    "                if state[i] == 0:\n",
    "                    state_ = copy.deepcopy(state)\n",
    "                    state_[i] = mark\n",
    "                    score = self.minimax(state_, mark * (-1), depth - 1, i, False, alpha, beta)\n",
    "                    maxScore = max(maxScore, [score[0], i], key=lambda x: x[0])\n",
    "                    alpha = max(alpha, maxScore[0])\n",
    "                    if alpha >= beta:\n",
    "                        break\n",
    "            return maxScore\n",
    "        else:\n",
    "            minScore = [float('inf'), node]\n",
    "            for i in range(9):\n",
    "                if state[i] == 0:\n",
    "                    state_ = copy.deepcopy(state)\n",
    "                    state_[i] = mark\n",
    "                    score = self.minimax(state_, mark * (-1), depth - 1, i, True, alpha, beta)\n",
    "                    minScore = min(minScore, [score[0], i], key=lambda x: x[0])\n",
    "                    beta = min(beta, minScore[0])\n",
    "                    if alpha >= beta:\n",
    "                        break\n",
    "            return minScore\n",
    "\n",
    "    def reset(self, is_human_first):\n",
    "        self.board = [0] * 9\n",
    "        self.current_turn = 1\n",
    "        self.player_mark = 1 if is_human_first else -1\n",
    "        if not is_human_first:\n",
    "            self.env_act()\n",
    "        return self.board.copy()\n",
    "\n",
    "    def check_win(self):\n",
    "        for pst in self.wining_position:\n",
    "            if str(self.board[pst[0]]) + str(self.board[pst[1]]) + str(self.board[pst[2]]) in ['111', '-1-1-1']:\n",
    "                if self.current_turn == self.player_mark:\n",
    "                    return 1, True\n",
    "                return -1, True\n",
    "        if 0 not in self.board:\n",
    "            return 0, True\n",
    "        return 0, False\n",
    "\n",
    "    def env_act(self):\n",
    "        action = self.minimax(self.board, self.current_turn, 25, 0, self.current_turn == 1, float('-inf'), float('inf'))\n",
    "        if self.board[action[1]] != 0:\n",
    "            raise Exception('Invalid action')\n",
    "        self.board[action[1]] = self.current_turn\n",
    "        reward, done = self.check_win()\n",
    "        self.current_turn = self.current_turn * -1\n",
    "        return reward, done\n",
    "\n",
    "    def step(self, action):\n",
    "        if self.board[action] != 0:\n",
    "            raise Exception('Invalid action')\n",
    "        self.board[action] = self.current_turn\n",
    "        reward, done = self.check_win()\n",
    "        self.current_turn = self.current_turn * -1\n",
    "        if done:\n",
    "            return self.board.copy(), reward, done, None\n",
    "        reward, done = self.env_act()\n",
    "        return self.board.copy(), reward, done, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_board(board):\n",
    "    mark = {1:'X', -1:'O', 0:' '}\n",
    "    for i in range(3):\n",
    "        print(end=' | ')\n",
    "        for j in range(3):\n",
    "            print(mark[board[3*i + j]], end=' | ')\n",
    "        print('\\n')\n",
    "    print('------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test agent again minimax player and hope that the agent will draw all matches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "result_vs_minimax = {1:0, 0:0, -1:0}\n",
    "env_test = Tictactoe_test_vs_minimax()\n",
    "for ep in range(1000):\n",
    "    done = False\n",
    "    reward = 0\n",
    "    state = env_test.reset(1)\n",
    "    is_first_move = True\n",
    "    print('game ' + str(ep) + ' start ---------------------')\n",
    "    while not done:\n",
    "        if is_first_move:\n",
    "            action = random.choice([i for i in range(len(state)) if state[i] == 0])\n",
    "            is_first_move = False\n",
    "        else:\n",
    "            action = agent.observe(state, [i for i in range(len(state)) if state[i] == 0])\n",
    "        print_board(state)\n",
    "        state, reward, done, _ = env_test.step(action)\n",
    "    result_vs_minimax[reward] += 1\n",
    "    print_board(state)\n",
    "    print('game ' + str(ep) + ' end ---------------------', reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Result:')\n",
    "print('win: ', result_vs_minimax[1])\n",
    "print('tie: ', result_vs_minimax[0])\n",
    "print('lose: ', result_vs_minimax[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test again random player"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "result_vs_random = {1:0, 0:0, -1:0}\n",
    "env_test = Tictactoe_v0()\n",
    "for ep in range(1000):\n",
    "    done = False\n",
    "    reward = 0\n",
    "    state = env_test.reset(1)\n",
    "    is_first_move = True\n",
    "    print('game ' + str(ep) + ' start ---------------------')\n",
    "    while not done:\n",
    "        if is_first_move:\n",
    "            action = random.choice([i for i in range(len(state)) if state[i] == 0])\n",
    "            is_first_move = False\n",
    "        else:\n",
    "            action = agent.observe(state, [i for i in range(len(state)) if state[i] == 0])\n",
    "        print_board(state)\n",
    "        state, reward, done, _ = env_test.step(action)\n",
    "    result_vs_random[reward] += 1\n",
    "    print_board(state)\n",
    "    print('game ' + str(ep) + ' end ---------------------', reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Result:')\n",
    "print('win: ', result_vs_random[1])\n",
    "print('tie: ', result_vs_random[0])\n",
    "print('lose: ', result_vs_random[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Result\n",
    "\n",
    "* VS MINIMAX PLAYER\n",
    "\n",
    "| training times | Wins | Draws | loses |   |\n",
    "|----------------|------|-------|-------|---|\n",
    "| 50000          | 0    | 1000  | 0     |   |\n",
    "\n",
    "* VS RANDOM PLAYER\n",
    "\n",
    "| training times | Wins | Draws | loses |   |\n",
    "|----------------|------|-------|-------|---|\n",
    "| 50000          | 809  | 190   | 1     |   |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}