File size: 27,409 Bytes
01c9c37 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 |
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Initial State: (0, 0)\n"
]
}
],
"source": [
"import numpy as np\n",
"\n",
"class Gridworld:\n",
" def __init__(self):\n",
" self.grid_size = 5\n",
" self.start_state = (0, 0)\n",
" self.goal_state = (4, 4)\n",
" self.obstacles = [(2, 2), (3, 3)]\n",
" self.state = self.start_state\n",
"\n",
" def reset(self):\n",
" self.state = self.start_state\n",
" return self.state\n",
"\n",
" def step(self, action):\n",
" actions = {\n",
" 0: (-1, 0), \n",
" 1: (1, 0), \n",
" 2: (0, -1), \n",
" 3: (0, 1) \n",
" }\n",
" next_state = (self.state[0] + actions[action][0],\n",
" self.state[1] + actions[action][1])\n",
"\n",
" if 0 <= next_state[0] < self.grid_size and 0 <= next_state[1] < self.grid_size:\n",
" self.state = next_state\n",
"\n",
" if self.state == self.goal_state:\n",
" return self.state, 100, True \n",
" elif self.state in self.obstacles:\n",
" return self.state, -10, False \n",
" else:\n",
" return self.state, -1, False \n",
"\n",
"env = Gridworld()\n",
"print(\"Initial State:\", env.reset())\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Q-Learning parameters\n",
"episodes = 500\n",
"alpha = 0.1 \n",
"gamma = 0.9 \n",
"epsilon = 0.2 \n",
"actions = [0, 1, 2, 3]\n",
"\n",
"# Initialize Q-table\n",
"Q_table = np.zeros((5, 5, len(actions)))\n",
"\n",
"# Q-Learning function\n",
"def train_gridworld(env):\n",
" for episode in range(episodes):\n",
" state = env.reset()\n",
" done = False\n",
"\n",
" while not done:\n",
" # Epsilon-greedy action selection\n",
" if np.random.uniform(0, 1) < epsilon:\n",
" action = np.random.choice(actions)\n",
" else:\n",
" action = np.argmax(Q_table[state[0], state[1], :])\n",
"\n",
" # Take action\n",
" next_state, reward, done = env.step(action)\n",
"\n",
" # Update Q-value\n",
" Q_table[state[0], state[1], action] = Q_table[state[0], state[1], action] + \\\n",
" alpha * (reward + gamma * np.max(Q_table[next_state[0], next_state[1], :]) -\n",
" Q_table[state[0], state[1], action])\n",
"\n",
" state = next_state\n",
"\n",
"train_gridworld(env)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['↓', '↓', '↓', '↓', '↓']\n",
"['→', '↓', '→', '→', '↓']\n",
"['→', '↓', '←', '→', '↓']\n",
"['→', '→', '↓', '→', '↓']\n",
"['→', '→', '→', '→', '↑']\n"
]
}
],
"source": [
"policy = np.argmax(Q_table, axis=2)\n",
"actions_mapping = {0: '↑', 1: '↓', 2: '←', 3: '→'}\n",
"\n",
"for row in policy:\n",
" print([actions_mapping[action] for action in row])\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"State space: Box([-1.2 -0.07], [0.6 0.07], (2,), float32)\n",
"Action space: Discrete(3)\n"
]
},
{
"data": {
"text/plain": [
"array([[[255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" ...,\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255]],\n",
"\n",
" [[255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" ...,\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255]],\n",
"\n",
" [[255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" ...,\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255]],\n",
"\n",
" ...,\n",
"\n",
" [[255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" ...,\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255]],\n",
"\n",
" [[255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" ...,\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255]],\n",
"\n",
" [[255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" ...,\n",
" [255, 255, 255],\n",
" [255, 255, 255],\n",
" [255, 255, 255]]], dtype=uint8)"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import gym\n",
"import matplotlib.pyplot as plt\n",
"\n",
"env = gym.make(\"MountainCar-v0\", render_mode=\"rgb_array\")\n",
"\n",
"print(\"State space:\", env.observation_space)\n",
"print(\"Action space:\", env.action_space)\n",
"\n",
"state = env.reset()\n",
"env.render()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\moham\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`. (Deprecated NumPy 1.24)\n",
" if not isinstance(terminated, (bool, np.bool8)):\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model saved as 'q_learning_model.pkl'\n"
]
}
],
"source": [
"import pickle\n",
"\n",
"state_bins = [20, 20]\n",
"action_space = env.action_space.n\n",
"Q_table = np.random.uniform(low=-1, high=1, size=(state_bins[0], state_bins[1], action_space))\n",
"\n",
"def discretize_state(state):\n",
" state_low = env.observation_space.low\n",
" state_high = env.observation_space.high\n",
" bins = [np.linspace(state_low[i], state_high[i], state_bins[i]) for i in range(len(state))]\n",
" state_indices = [np.digitize(state[i], bins[i]) - 1 for i in range(len(state))]\n",
" return tuple(state_indices)\n",
"\n",
"# Initialize Q-learning parameters\n",
"alpha = 0.1\n",
"gamma = 0.99\n",
"epsilon = 0.2\n",
"episodes = 5000\n",
"epsilon_decay = 0.995\n",
"\n",
"total_rewards = []\n",
"\n",
"# Train the agent\n",
"for episode in range(episodes):\n",
" state, _ = env.reset()\n",
" state = discretize_state(state)\n",
" done = False\n",
" total_reward = 0\n",
"\n",
" while not done:\n",
" # Epsilon-greedy action selection\n",
" if np.random.uniform(0, 1) < epsilon:\n",
" action = np.random.choice(action_space)\n",
" else:\n",
" action = np.argmax(Q_table[state])\n",
"\n",
" # Take action\n",
" next_state, reward, done, _, _ = env.step(action)\n",
" next_state = discretize_state(next_state)\n",
" total_reward += reward\n",
"\n",
" # Update Q-value\n",
" Q_table[state + (action,)] += alpha * (\n",
" reward + gamma * np.max(Q_table[next_state]) - Q_table[state + (action,)]\n",
" )\n",
" state = next_state\n",
"\n",
" # Decay epsilon\n",
" epsilon = max(0.01, epsilon * epsilon_decay)\n",
"\n",
" total_rewards.append(total_reward)\n",
"\n",
"# Save the model as a .pkl file\n",
"model_data = {\n",
" \"Q_table\": Q_table,\n",
" \"state_bins\": state_bins,\n",
" \"alpha\": alpha,\n",
" \"gamma\": gamma,\n",
" \"epsilon_decay\": epsilon_decay,\n",
"}\n",
"with open(\"q_learning_model.pkl\", \"wb\") as f:\n",
" pickle.dump(model_data, f)\n",
"\n",
"print(\"Model saved as 'q_learning_model.pkl'\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Total reward: -140.0\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAFeCAYAAAAYIxzjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwIklEQVR4nO3dd3RU5aL+8Wdm0iYBISGBACGhSEsAAQExtEBUWgARBRG4ENRrQVHxqNdyztF7LcfjAbGB7dAEBARFOkoQEAuIgDSDtAASIKEkkDqTmf37wwM/C2qASXYm+/tZK2u5AjPzLJlkP/Put9gMwzAEAAAsy252AAAAYC7KAAAAFkcZAADA4igDAABYHGUAAACLowwAAGBxlAEAACyOMgAAgMVRBgAAsDjKAAAAFkcZAADA4igDAABYHGUAAACLowwAAGBxlAEAACyOMgAAgMVRBgAAsDjKAAAAFkcZAADA4igDAABYHGUAAACLCzA7AAAAVmEYxgW/b7PZyjnJL1EGAAAoJx7PSe3c2UJhYR0UGtpBYWHtFRraVjZbkGy2ANlsgf/5Kt9yYDN+r6YAAACfcruztW1bzV99N0BOZys5na0UGtpKTmdLBQTUkMNR7fyXzVa2n90pAwAAlJMLl4HfCgqqr6CghgoObqjg4AYKCqqnwMB6Cgqqq6CgurLbQ32aizIAAEA5KW0Z+DWHI1wBATUVEBClwMAoBQU1VEhIc4WENJPT2UwBATUuKxdzBgAAqOA8ntPyeE6ruHi3JMlmC5LdHia7PVR2e5hiYv6l6tX7XfLzUwYAAKjgfppUGCybLVh2e7CCgxsrLOwahYa2V1hYewUFxVzW81MGAACoYByO6nI4IhQQEC6HI0IhIU3ldLaU05kgp7OFHI5qPn09ygAAAKayKygo9mdf9RQU1EBBQXEKDq6voKA42e3BZZqAMgAAQDmy2ULkdCYoJCRBTme8QkKaKyAgUg5HDQUERCggoIZstvLdIJgyAABAOTlzRnr22daaO3fhf+7/h/xnLgA7EAIAYAler3TyZJACA2uZHeUXOKgIAACLowwAAGBxlAEAACyOMgAAgMUxgRAAgMtw4MABnT17tlR/NycnR/n5+dq2bZvPczRp0kQhISGX9FgOKgIA4DL07t1bK1asMDuGtm3bppYtW17SY7lNAACAxVEGAACwOMoAAAAWRxkAAMDiKAMAAFgcZQAAAIujDAAAYHGUAQAALI4dCAEAKANVq1ZVlSpVFBgYKElyuVzKy8tTXl5eqR4fEBCgatWqyel0yuFwyOPxqLCwUGfOnJHb7fZpVsoAAAA+FBwcrCuvvFLx8fGqW7euqlatKsMwlJeXp4MHD2rRokUKCAhQcXHx7z5HjRo11LJlSzVq1EiRkZEKCQlRcXGxTp48qX379mnHjh3KysryWWbKAAAAPhISEqJrrrlGHTp0UGhoqGw22/k/q169ugIDAxUZGalmzZppw4YNKioq+s1z1KxZUykpKapbt64cDscvnrtu3bqKjo5Wo0aNtHTpUmVmZvokN3MGAADwkfr16ysxMVFhYWG/KAI/FxISosTERMXFxV3wz/v376969er9ogj8nMPhUN26dXXjjTf6KjZlAAAAX6hdu7ZuueUWBQcH/+nfDQ4O1uDBgxUdHX3+ew6HQ0OHDlXdunV/t0j8XFRUlEaNGvW7peFiUAYAALhMdrtdiYmJF3Vhdjgc6tSp0/kLf8OGDVWrVq1SFQFJstlsqlGjhpo2bXpJmX+OMgAAwGWy2Wxq0aLFRT8uISFBdvtPl+I6deqoevXqF/X4qlWrKjY29qJf99coAwAAWBxlAACAyzBt2jTt3r271MP7P2e32/XDDz/owIEDeuihhy7p9VNTU3XgwIHLul3A0kIAAC5DrVq1VKNGDTmdThUWFl7UY51Op+Li4uRwOJSZmXl+c6HSstvtioyMVP369S8y9a+e57IeDQAAZLfblZycfNGPS05OPj9noFmzZr9YXVAa5zYnulyUAQAALpPdblfz5s2VkJBQ6se0aNFCzZs3P397ISIiQomJiQoJCSnV4wMCAtS5c2dFRUVdUuafowwAAOADoaGh6tq1q2JiYv7079arV09dunRRaGjoL74fHx+vbt26ler1unTpolatWl1S1l+jDAAA4CORkZHq27evYmNj5XA4fjGp0GazyeFwqGbNmurbt68iIyMv+Bzt2rVTv3795HQ6fzMp0WazKSQkRL169VLHjh19lttmGIbhs2cDAMDizl1Wd+3apd27d+vkyZOSfrq/X6NGDf3tb3/TmjVr5Ha7VVRUJIfDIcMwZBiGPB6PXC6XnE6n8vLytGPHDv34448qLi5WaGioYmJidNVVVykiIkKSLmkFw4WwmgAAAB86d4FOSEhQ06ZNdfLkSR09elTHjh3T8ePHlZ2drZkzZyo3N1dHjhxRdHS0DMOQ1+tVXl6eDh8+rPj4eAUGBiogIOD8aMAVV1whSTpw4ICys7MVGBio+vXr+2Q7YsoAAAA+kp+fr82bN2vr1q06c+aMjh8/ruDgYLlcLp09e1Z16tSRy+VSTk7O+aOOr7jiCjkcjvMjBI0aNZLT6ZTL5VJxcbGKiop05swZnThxQm63Wy6XS3l5edq3b58aNGiguLg4NWrU6Px/h4WFXXRubhMAAHCJDMPQ5s2btWPHDn399dfKyMhQRESEatasqWuvvVYxMTGqUqWKQkNDFRQUpNDQUGVmZiohIeGiPtEbhqHCwsLzXwUFBcrJyVFhYaEyMjKUkZGhvXv3KiMjQ506dVJSUpISExPPjyb8GcoAAAClYBiG3G633G63Tp48qcWLF2vhwoXKyclR7969lZSUpJYtW8rpdMrhcCgwMPA3kwjLIlNJScn5r8LCQq1fv15paWnauHGjmjRpolmzZv3p81AGAAD4A16vVydOnFBGRoZWr16tPXv26NChQ+rdu7dSUlLUqFGj8xsHSb6b1Hepzl3WXS6XNm/erGuvvfZPH0MZAADgAs6ePat9+/bpu+++048//qhTp04pKipKycnJateunekXfV+iDAAA8DMFBQX65JNPtGLFCkVERCguLk7x8fFKSEg4v6SvsqEMAAAs79ylcPny5Zo5c6YiIiLUu3dvXXXVVapZs6aCgoJMTli2KAMAAMs6t7Z/xYoVeuedd9SsWTPdfffduvLKKxUYGPiLuQCVGWUAAGBJR48e1bZt27RkyRKVlJTovvvuU/PmzS1TAH6OMgAAsJTMzEx9/vnn2rNnj0pKSpSSkqI2bdr4ZCc/f0UZAABYgsvl0ieffKKFCxcqPj5eiYmJatOmjYKDg82OZjrKAACgUjMMQ/n5+XrmmWd09uxZDR06VFdffbXCwsIq1fLAy8HZBACASqmkpES5ublatWqVXnnlFf3P//yPevfuff7wH/x/lAEAQKXjdruVlpamDz74QA0bNtSKFStKvU+/FXGbAABQqRw8eFBz585VcXGxOnTooKSkJOYF/AlGBgAAlYJhGFqxYoU++eQTde3aVYmJiapVq5bZsfwCZQAA4NcMw9Dhw4f19NNPq1q1ahozZowaNGhg6aWCF4vbBAAAv+V2u7V//36NHz9eCQkJGjNmTJkfG1wZUQYAAH4pKytLa9eu1erVq5WamqoOHTqYHclvUQYAAH5n9+7d+vjjjxUWFqYhQ4YoMjLS7Eh+jTkDAAC/4fV6lZaWpjlz5ui2225TYmKinE6n2bH8HmUAAOAX3G63Zs6cqQ0bNujFF19UeHg4kwR9hDIAAKjQvF6vjh8/rn//+98KDg7W5MmTJYlJgj5EGQAAVFhFRUX64osvtHbtWrVu3Vr9+vWjBJQBygAAoELyer36+OOPtWLFCt17771q06aNAgK4bJUFVhMAACqkV199Vfn5+Ro8eLAaNWpkdpxKjTIAAKgwDMNQUVGRnn/+ecXExOi//uu/WC1QDhhvAQBUCB6PRz/88INmzZqlli1bauDAgQoKCjI7liVQBgAApjMMQ5s2bdJbb72lIUOG6LrrrmPZYDniNgEAwHSfffaZVq9ereTkZCUlJZkdx3IoAwAA0xiGoYULF2rr1q26/fbbFRsba3YkS6IMAABM4Xa7tXjxYu3evVt33HGHIiMj2UPAJJQBAEC5MgxDbrdbCxYs0OHDh5WamqqoqCizY1kaEwgBAOVu0qRJOnPmjMaOHavq1aubHcfyGBkAAJSb4uJi/fWvf1WbNm3Ur18/ValSxexIEGUAAFAODMNQQUGBnnvuOXXt2lXXXXcdWwtXIJQBAECZMgxDp0+f1tSpU9WwYUMNGDBAdrvd7Fj4GWoZAKBMZWVlacqUKYqJidHAgQPNjoMLoJoBAMpMVlaW3nzzTUVHR2vEiBFmx8HvYGQAAFAmjh8/rkmTJqlbt27q3r272XHwBygDAACfMgxDJ0+e1DvvvKPk5GR17tyZzYQqOMoAAMBnzhWB2bNnq3Xr1urSpQtFwA9QBgAAPpORkaE5c+aoYcOGSklJMTsOSokJhAAAnzh16pQmTpyounXrasiQIWbHwUVgnwEAwGU7e/asXnjhBSUnJ6tHjx7cGvAz3CYAAFwywzBUVFSkN954Q507d1ZSUhJFwA9RBgAAl8zlcmnWrFmKjIxU7969KQJ+ijkDAIBL4vV6NX36dOXk5Gj06NEUAT/GyAAA4JJMmDBBNptN999/P2cN+DkmEAIALtrkyZNlt9s1YsQIhYaGmh0Hl4mRAQBAqXk8Hi1atEgej0e33XabnE6n2ZHgA4zrAABKxePx6IsvvtC+fft00003qVq1aswTqCQoAwCAP2UYhjZt2qT169erf//+qlOnjtmR4EOUAQDAn1qyZIkmTpyogQMHqkmTJmbHgY8xZwAA8LsMw9DBgwc1f/58PfXUU2revLnZkVAGWE0AALggwzCUnZ2tf/zjH7r99tsVHx/PHIFKipEBAMAFnT17VtOnT1dycrISEhLMjoMyxJwBAMBvuFwuzZ49WzVr1tR1111ndhyUMUYGAAC/8eabb8put2vQoEEKDg42Ow7KGGUAAHCeYRh67rnnlJ6erjfffFNVqlQxOxLKAWUAACDpp02F1q1bp8LCQk2ePJkiYCHMGQAAyOv1aufOnVq3bp3uvPNOVa1a1exIKEeUAQCAsrKytGDBAvXu3Vv169c3Ow7KGWUAACzO5XJp4sSJSkxM1NVXX212HJiAMgAAFubxePTMM8+odevW6tGjhxwOh9mRYALKAABYVHFxsZ544gkdPXpUgwcPVmBgoNmRYBLKAABYkNvt1qpVqxQREaHXXntNdjuXAyvjXx8ALGj79u3atGmThg0bprCwMLPjwGSlLgOzZ88uyxwAgHKSlZWluXPnqn///qpbt67ZcVABlLoMZGdn67333pPX6y3LPACAMlRcXKyXXnpJycnJatWqFacQQtJFlIERI0YoIyND69evl8fjKctMAIAykJubq/Hjxys+Pl7XX389KwdwXqnLQEREhG6++WZ99tln2r9/vwzDKMtcAAAfKi4u1rvvvqucnByNHDmSEQH8wkVNIGzevLk6deqkmTNnKi8vr6wyAQB8bPXq1SosLNQTTzzBygH8xkW/I7p166b4+HhNnDiR0QEA8APp6enauHGjBg8erGrVqpkdBxXQRZeBwMBADRo0SC6XSy+//LLcbndZ5AIAXCbDMHTy5EnNmzdPSUlJuvLKK7k9gAu6pLGigIAAPfXUU/r22281f/58VhgAQAVUXFys6dOnKzY2Vl27duX2AH7XJb8zgoOD9eKLL2r79u3asWOHLzMBAHxg2rRpcrlcGjVqFCMC+EOXVRNr166tvn37atmyZTp69KivMgEALtPMmTO1detW3X///WZHgR+4rDLgcDjUvn17xcXFacGCBSouLvZVLgDAJTAMQ5s3b9aePXv00EMPKTQ01OxI8AOXfQMpKChIgwcP1rFjx7RkyRLmDwCASQzD0LFjx7Ry5UrdcMMNatKkCbcHUCo+mU3icDj07LPP6v3339eXX37pi6cEAFwkt9utjz76SNHR0erUqRNFAKXm06mlzz//vN59911t3rzZl08LAPgThmFo0aJFys7O1tChQ82OAz/j0zLQqFEjjRo1SsuWLdOPP/7oy6cGAPyB1atX69tvv9WYMWMUEhJidhz4GZ+WAYfDoU6dOqlBgwZavny5CgsLffn0AIBfMQxD33zzjd544w3de++9ioyMNDsS/JDPd6AIDAzUrbfeql27dmnTpk1sWQwAZejUqVOaNm2annzyScXExJgdB36qTLajcjgceuSRRzR37lzt3LmzLF4CACyvoKBACxcuVGJiolq0aMGEQVyyMtubsk6dOrrzzjs1ZcoUZWRklNXLAIAleTweff755zp9+rR69uyp4OBgsyPBj5XpRtWtWrXSgAED9Mwzzyg7O7ssXwoALCUjI0MLFizQkCFDmCeAy2Yzyvimvsvl0qxZs5Sbm6uxY8dyUAYAXKaSkhL17NlTU6ZMUVxcnNlxUAmU+ZU5MDBQKSkpcrvdWrt2rTweT1m/JABUWrm5uXr00Uf16KOPKjY21uw4qCTKvAzYbDZFRUWpZ8+eWrt2rTIyMlhhAACXoKCgQFOmTFFISIi6dOnChEH4TLmN2bdq1UqdO3fWK6+8wvkFAHCRDMPQpk2blJOTo7Fjx3IAEXyqXG/gd+/eXa1atdJLL71Uni8LAH4vKytLy5Yt08CBAxUdHW12HFQy5VoGHA6HRowYoaKiIs2bN4/5AwBQCi6XS5MnT1bHjh3VsmVLs+OgEir3qf1BQUG66667NGvWLKWlpTF/AAD+gMfj0cyZMxUcHKwBAwbI4XCYHQmVULmXAZvNptq1a+uJJ57Ql19+qaysrPKOAAB+Y82aNdq0aZMee+wxJgyizJi26P+qq65SkyZNtGjRIg40AoALWL9+vd577z2NGzeOIoAyZVoZCAkJUUpKig4fPqz169dzuwAAfubo0aNasWKFBg0apPr161MGUKbKfAfCP1NUVKSePXvqgw8+UM2aNc2MAgAVgtvt1gcffKATJ07o3nvvVUBAgNmRUMmZvjdwSEiI3n77bT366KPMHwBgeYZhaMuWLdqwYYNSU1MpAigXppcBSWrcuLH69u2rf/3rX8rMzDQ7DgCYZt++fZo1a5buueceVa1a1ew4sIgKUQbsdrt69uypKlWq6NNPP2X/AQCWlJubq/Hjx+u2225Ts2bNzI4DC6kQZUCSrrjiCqWmpmr37t1KT09nQiEASzEMQxMnTlRycrLat29vdhxYTIUpA5IUExOjvn37atasWTp9+jSFAIAleDwezZgxQ0ePHlW3bt1YOYByV6HKgM1mU6dOndS4cWO9++67KikpMTsSAJQpwzCUnp6u9PR03X///YqKiqIMoNxVqDJwTmpqqk6dOqWPPvrI7CgAUKYKCws1f/58de3aVQkJCWbHgUVVyDIgSQ8//LC2bt2q9evXmx0FAMqEYRiaOnWqoqKilJycbHYcWFiFLQORkZHq16+fxo8fr++++475AwAqFcMw9Omnn2rfvn0aOXKkgoKCzI4EC6uwZcBms6ljx44aMmSI1q5dq6KiIrMjAYDP7N+/X//+97/1v//7vwoLCzM7DiyuwpYB6adC0KdPH+Xl5WndunXsPwCgUjh69Khee+01PfjggwoNDTU7DlCxy4D00/4Dt956qz799FPt3r3b7DgAcFnOnj2ruXPnKiYmRgkJCbLbK/yvYViAX7wLGzZsqNTUVD311FMqLi42Ow4AXBKv16sdO3YoMzNTI0eO1BVXXGF2JECSn5QBSYqPj9fIkSP1t7/9TV6v1+w4AHDR8vLy9Oqrr+quu+5SVFSU2XGA8/ymDNhsNt1www2Kjo7W7NmzGSEA4Ffy8/P1zDPPKDU1VQ0bNjQ7DvALflMGJMnpdColJUXr1q3Tt99+y3JDAH7B7XZr2rRpatiwoa6//np2GESF41dlQPrpuONbb71Vq1ev1unTp82OAwB/Ki0tTVlZWUpNTaUIoELyuzIgSZ06dVJ0dLTmzZvHckMAFdrWrVu1ePFi3XTTTXI6nWbHAS7IL8tAcHCwRowYwXbFACoswzB08uRJvf/++0pMTFSLFi0YFUCF5ZdlQPqpELz++uv6+9//rszMTLPjAMAvGIahL7/8UoGBgbrtttvkcDjMjgT8Lr8tA5LkcDj04osvatKkSTp27JjZcQDgvM2bN2vlypV64IEHGBFAhefXZcBms6l169Zq2rSpZsyYodzcXLMjAYAOHjyoqVOnasyYMewnAL/g12VA+ul2Qd++fXXy5Elt2bLF7DgALM7j8ej555/X8OHD1axZM7PjAKXi92VAksLDwzVy5EgtWrRIR44cYf8BAKYoKSnR9OnT1b59e7Vp04bbA/AblaIM2Gw2xcfHKykpSW+99ZYKCgrMjgTAYjwej9auXaudO3cqOTlZISEhZkcCSq1SlIFz+vfvr+rVq2vKlClmRwFgMadOndK8efOUkpKiBg0amB0HuCiVqgxI0t13363MzEytXLnS7CgALMLr9Wrq1Km69tprlZSUZHYc4KJVujLgdDp1++236/PPP9fu3buZPwCgTBmGofnz56uwsFBDhgxhngD8UqUrAzabTY0aNVLHjh01ffp0zi8AUKa+++47LV68WI899hjbDcNvVboyIP1UCLp37y6n06nVq1erpKTE7EgAKqGsrCy9/fbb+utf/6rg4GCz4wCXrFKWAUkKCwtTamqqtmzZoi1btnC7AIBP5eTkaN68eerRo4fi4uK4PQC/VmnLgCTFxMRoxIgRevnll9mdEIDPuN1uLVmyREePHlVycjKjAvB7lboMSFKzZs1055136tFHH5XX6zU7DgA/ZxiGTp8+rcWLF+uee+5ReHi42ZGAy1bpy4Akde7cWW3bttW0adPkdrvNjgPAj+Xn5+vvf/+7HnjgAdWtW9fsOIBPWKIMBAYGauDAgcrMzNSGDRsYIQBwSYqKivTmm2+qQ4cOSkxMZJ4AKg1LlAFJqlWrlpKSkrRgwQIdOXLE7DgA/NCSJUtUVFSk4cOHmx0F8CnLlAFJuvrqq3XVVVdp/vz5crlcZscB4Ee2bNmiXbt2afjw4QoICDA7DuBTlioDTqdTt956q7Kzs7Vy5UqWGwL4U4ZhKCsrS59++qk6d+6s2NhYbg+g0rFUGZCkkJAQPfvss3rttde0b98+s+MAqODcbrfee+89FRYWqnv37rLbLfdrExZgyXe1zWbTK6+8oldffVU//vij2XEAVGDffPONDh06pHHjxjEigErLsmWgcePGuu666/Thhx8qJyfH7EgAKqDvv/9ec+bM0dixY1W1alWz4wBlxpJlQJICAgKUnJyskpISffbZZyw3BPALZ8+e1YQJEzR69Gg1atTI7DhAmbJsGZB+Or+gZ8+emjlzpr7//nsmFAKQJHk8Hk2ePFk9e/ZUy5YtzY4DlDlLlwFJio+P14MPPshxxwAkSSUlJUpLS1NAQIC6d+8uh8NhdiSgzFm+DNhsNnXp0kWtWrXS66+/znHHgIUZhqH09HStWbNGPXv2VI0aNZg0CEuwfBk4Z9iwYZKkOXPmmJwEgFncbrfeeustdejQQQkJCWbHAcoNZeBn/vu//1t79+7Vl19+yfwBwGIMw9CkSZMUHx+vPn36mB0HKFeUgf+w2WyqVauW+vXrp1WrVunIkSMUAsAivF6vVq5cqcOHD+v2229XUFCQ2ZGAckUZ+Bmbzaarr75aderU0QcffKDCwkKzIwEoB7t379bChQv15JNPUgRgSZSBC0hJSdGhQ4e0atUqRgeASu748eP66KOPNHz4cFWvXt3sOIApKAMXEB0drXHjxiktLU3btm0zOw6AMlJYWKgVK1YoJiZGbdu25dwBWBbv/N9Rr149Pfzww3rqqad09uxZs+MA8DHDMLR69WqtW7dON954o0JDQ82OBJjGZjAO/rsMw9CaNWu0bNkyPfvsswoODjY7EgAfOXXqlIYMGaKZM2eqVq1aZscBTMXIwB+w2Wzq0KGDGjdurIULF6q4uNjsSAB8IDs7W88884z+7//+jyIAiDLwp8LCwtSnTx/t2bNHW7Zs4UAjwM/l5eXpvffeU2Jiojp06GB2HKBCoAyUQkxMjLp3767Jkydz3DHg55YsWaLAwED169ePCYPAf/CTUEodOnRQ//799dJLLzE6APghwzC0fft2paenq1+/fnI6nWZHAioMykApBQYGauDAgYqIiNDUqVM50AjwI4Zh6OjRo5o3b56Sk5MVFxfHAUTAz1AGLoLdbtcDDzyg9PR0rV692uw4AErJ6/XqueeeU5UqVdSlSxeKAPArlIGLFBgYqDFjxmjNmjXatWuX2XEAlMKcOXMUHh6uxx57zOwoQIVEGbhINptNMTExuu6667R8+XJlZ2ebHQnAH1i1apXS09M1btw4s6MAFRZl4BIEBASoc+fOstvtWrBggVwul9mRAPyKYRjas2ePPvvsM40YMULh4eFmRwIqLMrAJQoKCtKDDz6oNWvWaMOGDRxoBFQwubm5WrBggbp166bGjRszTwD4A5SBy2Cz2fTqq69q2rRpzB8AKhC3262lS5cqNDRU3bp1owgAf4IycJlq1qyp+++/X7NmzdL+/fvNjgNYnmEYmjVrltasWaMhQ4ZwpghQCpQBH2jZsqWSkpI0Z84cnTlzxuw4gKXt2bNHS5cu1SOPPMK5A0ApUQZ8wOFwKCkpSdWqVdPixYvZoRAwSX5+vsaNG6eJEyeqSZMmZscB/AZlwEcCAwN1yy23aOfOnVq/fj2FAChneXl5mjBhgh544AFFR0ebHQfwK5QBH7HZbKpZs6ZSUlL01ltvaefOnWZHAiyjqKhIy5cvV506ddSxY0c5HA6zIwF+hTLgY4mJiRo9erQmTJig3Nxcs+MAlZ7X69XWrVuVnp6u3r17q2rVqmZHAvyOzWCBvM8ZhqGVK1dqxYoVevnll1nWBJShwsJC3XLLLXrjjTcUFxdndhzALzEyUEaSkpLUokULTZs2jRMOgTJSVFSkQYMGaezYsYqNjTU7DuC3KANlwGazKSQkRH379tWJEyf01VdfyePxmB0LqFTOnDmjiRMnavTo0br++usZgQMuA2WgDNWuXVu9evXSqlWrdPDgQbYsBnykqKhIixcvVnh4uFJSUigCwGWiDJSxli1bqlOnTnr66acZHQB8wDAMbdiwQRkZGbr55psVEhJidiTA71EGykH37t3Vp08fPfHEE4wOAJfBMAxlZmZq4cKFuuWWW1SjRg2zIwGVAmWgHAQGBurmm29W/fr19e6778rtdpsdCfBLp0+f1gsvvKCbbrpJjRs3NjsOUGlQBspJQECAhg0bplOnTumzzz6jEAAX6cyZM3rkkUcUFRWlLl26ME8A8CH2GShnBw4c0IwZMzRo0CAlJCTwCw0oBZfLpXfeeUc2m01333237HY+xwC+xE9UOWvQoIF69eqlGTNmsEMhUEqLFy+Wy+XSyJEjKQJAGeCnygTt2rVTQkKC7rvvPlYYAH/AMAxt3rxZO3fu1M0336zQ0FCzIwGVEmXABA6HQyNGjFDTpk319NNPq6ioyOxIQIVjGIaOHDmijz76SCkpKYqJieG2GlBGKAMmsdvtevzxx1W1alV9/PHHKi4uNjsSUKEcP35cb7zxhjp27Ki2bdtSBIAyRBkwUUBAgO68804dOHBA69atYw8C4D8KCwv14osv6sorr1Tfvn3NjgNUepQBk4WHh2vw4MFau3atfvjhB7PjABXC66+/rlatWmnUqFFmRwEsgTJQAdSvX1833nijJk+erNOnT5sdBzCN1+vVhx9+qJCQEA0aNIiVA0A54SetArDb7br66qt1zTXX6JFHHtHJkyfNjgSUO6/Xq02bNik9PV0DBw5U1apVmScAlBPKQAVhs9k0dOhQNWnSRBMnTmQPAliKYRjau3evli9frj59+rByAChnlIEK5qGHHlKDBg00b948lhzCMg4dOqTx48frhhtuUOvWrc2OA1gOZaCCCQwM1JAhQ5Sfn6/ly5ezwgCVXkFBgR577DGNHDlS1157rdlxAEuiDFRAYWFhGjFihL744gtt27aNQoBKy+1269lnn9Xtt9+ujh07mh0HsCzKQAUVERGh++67T5MmTVJ6errZcQCfKyoq0uzZs9W0aVNOIQRMRhmooGw2m+rXr6/hw4fr+eef1+bNm82OBPhMSUmJPvnkE+Xm5iolJUUhISGUAcBEAWYHwB/r3Lmz8vLy9MEHH6h69epq2LCh2ZGAy7Z69Wrt2LFDo0aNUo0aNcyOA1iezeCGdIXn8Xi0bt06ffPNN0pNTVVkZCSfouCXDMPQsmXLNGXKFE2aNEm1atUyOxIAcZvALzgcDnXt2lVxcXFasGCB8vPzmVQIv+P1erVlyxbNnj1bb7zxhmrWrGl2JAD/QRnwEw6H4/ySw/nz58vr9ZodCSg1wzB04MABffTRR/rrX/+q6OhoRreACoQy4Gcefvhhff/995oxY4bZUYBSO3HihGbOnKlevXqpWbNmZscB8CuUAT/05JNP6vDhw3r77bfNjgL8KbfbrX/84x+69tprlZiYaHYcABdAGfBDVatW1ZgxY1RUVKQPP/yQ+QOosAzD0H333ac+ffooOTmZWwNABUUZ8EM2m00REREaMmSI9uzZo88//1wej8fsWMAvFBQUaPTo0WrcuLG6d+8uh8NhdiQAv4OlhX7uwIED5+/FtmvXjk9eqBByc3M1b948Va9eXQMGDFBQUJDZkQD8AUYG/FyDBg10yy23aNGiRfriiy/MjgOooKBAixYtUmhoqHr27EkRAPwAZaASaNasmYYOHarJkydr8eLFZseBhXm9Xs2ePVtut1t9+/bVFVdcYXYkAKVAGagkmjdvrscff1ybNm3ipEOYwuv1atq0acrLy9OQIUNUvXp1syMBKCXOJqgkbDabEhISZBiGli5dquDgYDVu3Fh2O30PZa+wsFBvvfWWdu7cqUmTJikwMNDsSAAuAleKSsRms6lly5bq1q2bFi1apL179zJCgDJXUFCgZcuWqbCwUP/85z8pAoAfYjVBJfXVV19p1apVuuGGG3TNNdeYHQeVlMvl0ooVK3T8+HH179+fg4cAP8XIQCV17bXXql+/fho/frzS0tLMjoNKyDAMffzxxzp48KAGDBhAEQD8GCMDlZhhGNq2bZs++OADDR8+XE2bNmUfAvhESUmJ3n//fR06dEgPPvigwsLCzI4E4DJQBio5wzC0ZcsWrVy5UgMHDlSTJk2YVIjLkp+fr9dff105OTl65pln2EcAqAS4KlRyNptNbdu2VVJSkj7++GNt377d7EjwY3l5efr444/l9Xr18MMPUwSASoKRAQvZsmWLli1bprZt26p3795mx4GfcblcmjVrliSpb9++qlmzpsmJAPgK+wxYSOvWreV0OvXPf/5ThmGoT58+ZkeCH3n55ZcVHR2tAQMGsKEQUMkwMmAxhmFo9+7deueddzRgwAB17tyZOQT4Q0VFRXr22WeVkJCgQYMGcWsAqIQoAxZkGIYOHTqkd999Vz169FC3bt0oBLig06dP67nnnlNCQoKGDx/OhkJAJUUZsLDMzEy9/fbbatGihW6++Waz46ACMQxDx44d08yZMxUeHq5bbrlF1apVMzsWgDJCGbC4nJwcvffeeyoqKtK9997LenFIkg4ePKhXXnlFvXr1UlJSErcGgEqOMmBxhmGooKBAc+fO1eHDh/XAAw+oWrVqbE5kUYZhaMeOHRo/frwee+wxNWvWjPcCYAGUAcgwDHk8Hs2bN08HDx7U7bffrqioKC4CFlNSUqL58+dr6dKlevHFF1W7dm3eA4BFUAbwCwsWLND27ds1bNgwNW7c2Ow4KCfFxcVKS0vTkiVLdM8996hly5ZmRwJQjigD+I1169Zp9erV6tKli5KTk82OgzLmcrk0depUFRYWauDAgYqLizM7EoByRhnAb5y7bzx9+nS1bdtWN910k0JCQsyOhTJw/PhxvfDCC2rXrp369OmjiIgIsyMBMAFlABfk9Xp15MgRTZ06VZGRkRo9ejSFoBIxDEPr16/Xu+++qzvvvFMdOnRgxQBgYZQB/K5zEwsnT56srKwsjRs3TuHh4WbHwmUwDENer1ebNm3Syy+/rDvvvFM9evRgoiBgcZQBlMqiRYu0bt063XbbbUpISFBwcLDZkXAJcnNzlZaWpq+++kqpqamKj483OxKACoAygFI592lyzpw5uvrqqzVgwABVqVLF7Fi4CHv37tWiRYvkcrl0xx13KDIy0uxIACoIygBKzev16uDBg1q8eLHy8vI0btw45hH4ibS0NC1fvlw9evRQr169OIsCwC/wGwGlZrfb1aBBA40ePVoJCQnq37+/9u3bJ6/Xa3Y0/A6Xy6VZs2ZpwoQJSklJ0Q033EARAPAbjAzgkpxbbfCXv/xF/fr104033qiwsDAmolUQXq9X2dnZeuutt1RcXKwnnnhCoaGh/PsAuCDKAC5LZmamXnrpJUVFRemmm25SkyZN+ORpslOnTmnjxo365JNP1KlTJw0aNMjsSAAqOMoALltWVpZWrlypffv2cRyyiQzDUFZWliZNmqT8/HyNGjVK8fHxlDMAf4oyAJ/54YcftHTpUqWnp+svf/kLZxuUs08++URTpkxR7969dcMNN6h27dpmRwLgJygD8JlzxyF/9dVXmjhxosaOHavu3bsrICCAe9VlxOv16syZM5o4caJycnL0+OOPKzw8nN0EAVwUygB86tzbadOmTXr++efVpk0bjRgxQnXr1uUC5UOGYejs2bP67LPP9M4772jw4MG69dZb+X8M4JJQBlCm5s+fr/Xr16tDhw7q2rWrYmJizI7k9zwejzZv3qx169Zp3759GjZsmDp16mR2LAB+jDKAMmUYhg4dOqQlS5Zo79696tSpk/r168d2xpfowIEDmjt3rlwul1q3bq3OnTtz0iCAy0YZQLkoKCjQjh07tGrVKqWnp2vs2LFq166d2bH8RmFhoaZOnaqNGzeqf//+uuaaa1SnTh3mYgDwCcoAyo1hGCoqKtLu3bs1YcIE1a1bV3fccYdiY2OZZHgBXq9XLpdLGzZs0IQJE1S/fn3dddddatKkiQICAsyOB6ASoQyg3J0rBQsWLNCMGTPUt29f9ezZU7GxsQoNDTU7num8Xq9OnDihXbt2aenSpcrPz9f999+vZs2aSRKlCYDPUQZgqu3bt+vTTz/V8ePHFRcXpzZt2qhly5aWPRExMzNTGzZs0Jdffqn8/HwNGjRInTt3Zo4FgDJFGUCFcOjQIX399dfatWuXTp06pZ49e6pXr15yOBxmRysXmZmZWrhwoTIyMlS7dm01adJEbdu2ZeMgAOWCMoAKw+PxKDs7W2lpadq0aZN27NihUaNGqW/fvqpWrZqkyjFE/vMfuf379+u1117TgQMHlJKSosTERMXGxqpq1aomJgRgNZQBVDgej0dut1snTpzQ22+/rY0bN6pp06a67777VLt2bYWEhPjlBLpzcyWKioq0detWvffeezp8+LB69eqlYcOGqUaNGkykBGAKygAqvMzMTL3++uv65ptv1KpVK7Vv317NmjVTeHi46tatW6GLwbktmo8dO6Zjx47p008/1caNGxUbG6uhQ4eqU6dOFTo/AGugDMBvuN1ubdy4UV9//bWysrKUlZWl5s2bq3nz5mrcuLHi4uLkdDrNjilJOnnypLZv364DBw4oOztbeXl5ys/PV7169dStWze1adPG7IgAcB5lAH7n3NK7LVu2nC8Fx44dU15enpxOp5KSktSqVSvFxMSUy6duwzDkdru1detWpaena+vWrSooKFB+fr4aN26stm3bqlGjRmrQoIGCg4O5DQCgwqEMwK+dO7DnzJkzOnHihObOnav8/Hzt379fubm5io+P1xVXXKEePXqoadOmql279i+W6ZX2wvzzHxOXy6Vdu3Zpz5492rVrl3bu3Kk9e/YoNjb2/Kf+evXqyel0qnr16kwGBFDhUQZQaRiGIY/HI8Mw5PV6lZubq++++06zZ8+Wy+XS0aNHdfLkSYWHh6u4uFgJCQmqWbOmqlSpoipVquj48eOKjo5WUFCQ3G633G63du7cqaCgIBUVFSkrK0vHjx9Xbm6u6tWrp/bt2yshIUEJCQlq0qSJgoKCZLfbZbfbZbPZGAEA4DcoA7AUl8ul48eP6+uvv1ZAQIA8Ho/y8/OVl5en3bt3KyIiQtWqVVNgYKACAgJ07NgxNWjQQA0bNlRUVJSioqJUvXr18xd8AKgMKAMAAFic3ewAAADAXJQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALC4/weJ1hqyIXcPWwAAAABJRU5ErkJggg==",
"text/plain": [
"<Figure size 640x480 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from IPython.display import display, clear_output\n",
"import time\n",
"\n",
"# Initialize environment\n",
"state, _ = env.reset()\n",
"state = discretize_state(state)\n",
"done = False\n",
"total_reward = 0\n",
"\n",
"# Test the policy\n",
"while not done:\n",
" action = np.argmax(Q_table[state])\n",
" next_state, reward, done, truncated, _ = env.step(action)\n",
" state = discretize_state(next_state)\n",
" total_reward += reward\n",
"\n",
" # Render frame\n",
" frame = env.render()\n",
" plt.imshow(frame)\n",
" plt.axis(\"off\")\n",
" display(plt.gcf())\n",
" clear_output(wait=True)\n",
" time.sleep(0.3) \n",
"\n",
"print(\"Total reward:\", total_reward)\n",
"env.close()\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.9"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
|