{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initial State: (0, 0)\n" ] } ], "source": [ "import numpy as np\n", "\n", "class Gridworld:\n", " def __init__(self):\n", " self.grid_size = 5\n", " self.start_state = (0, 0)\n", " self.goal_state = (4, 4)\n", " self.obstacles = [(2, 2), (3, 3)]\n", " self.state = self.start_state\n", "\n", " def reset(self):\n", " self.state = self.start_state\n", " return self.state\n", "\n", " def step(self, action):\n", " actions = {\n", " 0: (-1, 0), \n", " 1: (1, 0), \n", " 2: (0, -1), \n", " 3: (0, 1) \n", " }\n", " next_state = (self.state[0] + actions[action][0],\n", " self.state[1] + actions[action][1])\n", "\n", " if 0 <= next_state[0] < self.grid_size and 0 <= next_state[1] < self.grid_size:\n", " self.state = next_state\n", "\n", " if self.state == self.goal_state:\n", " return self.state, 100, True \n", " elif self.state in self.obstacles:\n", " return self.state, -10, False \n", " else:\n", " return self.state, -1, False \n", "\n", "env = Gridworld()\n", "print(\"Initial State:\", env.reset())\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Q-Learning parameters\n", "episodes = 500\n", "alpha = 0.1 \n", "gamma = 0.9 \n", "epsilon = 0.2 \n", "actions = [0, 1, 2, 3]\n", "\n", "# Initialize Q-table\n", "Q_table = np.zeros((5, 5, len(actions)))\n", "\n", "# Q-Learning function\n", "def train_gridworld(env):\n", " for episode in range(episodes):\n", " state = env.reset()\n", " done = False\n", "\n", " while not done:\n", " # Epsilon-greedy action selection\n", " if np.random.uniform(0, 1) < epsilon:\n", " action = np.random.choice(actions)\n", " else:\n", " action = np.argmax(Q_table[state[0], state[1], :])\n", "\n", " # Take action\n", " next_state, reward, done = env.step(action)\n", "\n", " # Update Q-value\n", " Q_table[state[0], state[1], action] = Q_table[state[0], state[1], action] + \\\n", " alpha * (reward + gamma * np.max(Q_table[next_state[0], next_state[1], :]) -\n", " Q_table[state[0], state[1], action])\n", "\n", " state = next_state\n", "\n", "train_gridworld(env)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['↓', '↓', '↓', '↓', '↓']\n", "['→', '↓', '→', '→', '↓']\n", "['→', '↓', '←', '→', '↓']\n", "['→', '→', '↓', '→', '↓']\n", "['→', '→', '→', '→', '↑']\n" ] } ], "source": [ "policy = np.argmax(Q_table, axis=2)\n", "actions_mapping = {0: '↑', 1: '↓', 2: '←', 3: '→'}\n", "\n", "for row in policy:\n", " print([actions_mapping[action] for action in row])\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "State space: Box([-1.2 -0.07], [0.6 0.07], (2,), float32)\n", "Action space: Discrete(3)\n" ] }, { "data": { "text/plain": [ "array([[[255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " ...,\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255]],\n", "\n", " [[255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " ...,\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255]],\n", "\n", " [[255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " ...,\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255]],\n", "\n", " ...,\n", "\n", " [[255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " ...,\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255]],\n", "\n", " [[255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " ...,\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255]],\n", "\n", " [[255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " ...,\n", " [255, 255, 255],\n", " [255, 255, 255],\n", " [255, 255, 255]]], dtype=uint8)" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import gym\n", "import matplotlib.pyplot as plt\n", "\n", "env = gym.make(\"MountainCar-v0\", render_mode=\"rgb_array\")\n", "\n", "print(\"State space:\", env.observation_space)\n", "print(\"Action space:\", env.action_space)\n", "\n", "state = env.reset()\n", "env.render()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\moham\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python311\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`. (Deprecated NumPy 1.24)\n", " if not isinstance(terminated, (bool, np.bool8)):\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Model saved as 'q_learning_model.pkl'\n" ] } ], "source": [ "import pickle\n", "\n", "state_bins = [20, 20]\n", "action_space = env.action_space.n\n", "Q_table = np.random.uniform(low=-1, high=1, size=(state_bins[0], state_bins[1], action_space))\n", "\n", "def discretize_state(state):\n", " state_low = env.observation_space.low\n", " state_high = env.observation_space.high\n", " bins = [np.linspace(state_low[i], state_high[i], state_bins[i]) for i in range(len(state))]\n", " state_indices = [np.digitize(state[i], bins[i]) - 1 for i in range(len(state))]\n", " return tuple(state_indices)\n", "\n", "# Initialize Q-learning parameters\n", "alpha = 0.1\n", "gamma = 0.99\n", "epsilon = 0.2\n", "episodes = 5000\n", "epsilon_decay = 0.995\n", "\n", "total_rewards = []\n", "\n", "# Train the agent\n", "for episode in range(episodes):\n", " state, _ = env.reset()\n", " state = discretize_state(state)\n", " done = False\n", " total_reward = 0\n", "\n", " while not done:\n", " # Epsilon-greedy action selection\n", " if np.random.uniform(0, 1) < epsilon:\n", " action = np.random.choice(action_space)\n", " else:\n", " action = np.argmax(Q_table[state])\n", "\n", " # Take action\n", " next_state, reward, done, _, _ = env.step(action)\n", " next_state = discretize_state(next_state)\n", " total_reward += reward\n", "\n", " # Update Q-value\n", " Q_table[state + (action,)] += alpha * (\n", " reward + gamma * np.max(Q_table[next_state]) - Q_table[state + (action,)]\n", " )\n", " state = next_state\n", "\n", " # Decay epsilon\n", " epsilon = max(0.01, epsilon * epsilon_decay)\n", "\n", " total_rewards.append(total_reward)\n", "\n", "# Save the model as a .pkl file\n", "model_data = {\n", " \"Q_table\": Q_table,\n", " \"state_bins\": state_bins,\n", " \"alpha\": alpha,\n", " \"gamma\": gamma,\n", " \"epsilon_decay\": epsilon_decay,\n", "}\n", "with open(\"q_learning_model.pkl\", \"wb\") as f:\n", " pickle.dump(model_data, f)\n", "\n", "print(\"Model saved as 'q_learning_model.pkl'\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total reward: -140.0\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAFeCAYAAAAYIxzjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwIklEQVR4nO3dd3RU5aL+8Wdm0iYBISGBACGhSEsAAQExtEBUWgARBRG4ENRrQVHxqNdyztF7LcfjAbGB7dAEBARFOkoQEAuIgDSDtAASIKEkkDqTmf37wwM/C2qASXYm+/tZK2u5AjPzLJlkP/Put9gMwzAEAAAsy252AAAAYC7KAAAAFkcZAADA4igDAABYHGUAAACLowwAAGBxlAEAACyOMgAAgMVRBgAAsDjKAAAAFkcZAADA4igDAABYHGUAAACLowwAAGBxlAEAACyOMgAAgMVRBgAAsDjKAAAAFkcZAADA4igDAABYHGUAAACLCzA7AAAAVmEYxgW/b7PZyjnJL1EGAAAoJx7PSe3c2UJhYR0UGtpBYWHtFRraVjZbkGy2ANlsgf/5Kt9yYDN+r6YAAACfcruztW1bzV99N0BOZys5na0UGtpKTmdLBQTUkMNR7fyXzVa2n90pAwAAlJMLl4HfCgqqr6CghgoObqjg4AYKCqqnwMB6Cgqqq6CgurLbQ32aizIAAEA5KW0Z+DWHI1wBATUVEBClwMAoBQU1VEhIc4WENJPT2UwBATUuKxdzBgAAqOA8ntPyeE6ruHi3JMlmC5LdHia7PVR2e5hiYv6l6tX7XfLzUwYAAKjgfppUGCybLVh2e7CCgxsrLOwahYa2V1hYewUFxVzW81MGAACoYByO6nI4IhQQEC6HI0IhIU3ldLaU05kgp7OFHI5qPn09ygAAAKayKygo9mdf9RQU1EBBQXEKDq6voKA42e3BZZqAMgAAQDmy2ULkdCYoJCRBTme8QkKaKyAgUg5HDQUERCggoIZstvLdIJgyAABAOTlzRnr22daaO3fhf+7/h/xnLgA7EAIAYAler3TyZJACA2uZHeUXOKgIAACLowwAAGBxlAEAACyOMgAAgMUxgRAAgMtw4MABnT17tlR/NycnR/n5+dq2bZvPczRp0kQhISGX9FgOKgIA4DL07t1bK1asMDuGtm3bppYtW17SY7lNAACAxVEGAACwOMoAAAAWRxkAAMDiKAMAAFgcZQAAAIujDAAAYHGUAQAALI4dCAEAKANVq1ZVlSpVFBgYKElyuVzKy8tTXl5eqR4fEBCgatWqyel0yuFwyOPxqLCwUGfOnJHb7fZpVsoAAAA+FBwcrCuvvFLx8fGqW7euqlatKsMwlJeXp4MHD2rRokUKCAhQcXHx7z5HjRo11LJlSzVq1EiRkZEKCQlRcXGxTp48qX379mnHjh3KysryWWbKAAAAPhISEqJrrrlGHTp0UGhoqGw22/k/q169ugIDAxUZGalmzZppw4YNKioq+s1z1KxZUykpKapbt64cDscvnrtu3bqKjo5Wo0aNtHTpUmVmZvokN3MGAADwkfr16ysxMVFhYWG/KAI/FxISosTERMXFxV3wz/v376969er9ogj8nMPhUN26dXXjjTf6KjZlAAAAX6hdu7ZuueUWBQcH/+nfDQ4O1uDBgxUdHX3+ew6HQ0OHDlXdunV/t0j8XFRUlEaNGvW7peFiUAYAALhMdrtdiYmJF3Vhdjgc6tSp0/kLf8OGDVWrVq1SFQFJstlsqlGjhpo2bXpJmX+OMgAAwGWy2Wxq0aLFRT8uISFBdvtPl+I6deqoevXqF/X4qlWrKjY29qJf99coAwAAWBxlAACAyzBt2jTt3r271MP7P2e32/XDDz/owIEDeuihhy7p9VNTU3XgwIHLul3A0kIAAC5DrVq1VKNGDTmdThUWFl7UY51Op+Li4uRwOJSZmXl+c6HSstvtioyMVP369S8y9a+e57IeDQAAZLfblZycfNGPS05OPj9noFmzZr9YXVAa5zYnulyUAQAALpPdblfz5s2VkJBQ6se0aNFCzZs3P397ISIiQomJiQoJCSnV4wMCAtS5c2dFRUVdUuafowwAAOADoaGh6tq1q2JiYv7079arV09dunRRaGjoL74fHx+vbt26ler1unTpolatWl1S1l+jDAAA4CORkZHq27evYmNj5XA4fjGp0GazyeFwqGbNmurbt68iIyMv+Bzt2rVTv3795HQ6fzMp0WazKSQkRL169VLHjh19lttmGIbhs2cDAMDizl1Wd+3apd27d+vkyZOSfrq/X6NGDf3tb3/TmjVr5Ha7VVRUJIfDIcMwZBiGPB6PXC6XnE6n8vLytGPHDv34448qLi5WaGioYmJidNVVVykiIkKSLmkFw4WwmgAAAB86d4FOSEhQ06ZNdfLkSR09elTHjh3T8ePHlZ2drZkzZyo3N1dHjhxRdHS0DMOQ1+tVXl6eDh8+rPj4eAUGBiogIOD8aMAVV1whSTpw4ICys7MVGBio+vXr+2Q7YsoAAAA+kp+fr82bN2vr1q06c+aMjh8/ruDgYLlcLp09e1Z16tSRy+VSTk7O+aOOr7jiCjkcjvMjBI0aNZLT6ZTL5VJxcbGKiop05swZnThxQm63Wy6XS3l5edq3b58aNGiguLg4NWrU6Px/h4WFXXRubhMAAHCJDMPQ5s2btWPHDn399dfKyMhQRESEatasqWuvvVYxMTGqUqWKQkNDFRQUpNDQUGVmZiohIeGiPtEbhqHCwsLzXwUFBcrJyVFhYaEyMjKUkZGhvXv3KiMjQ506dVJSUpISExPPjyb8GcoAAAClYBiG3G633G63Tp48qcWLF2vhwoXKyclR7969lZSUpJYtW8rpdMrhcCgwMPA3kwjLIlNJScn5r8LCQq1fv15paWnauHGjmjRpolmzZv3p81AGAAD4A16vVydOnFBGRoZWr16tPXv26NChQ+rdu7dSUlLUqFGj8xsHSb6b1Hepzl3WXS6XNm/erGuvvfZPH0MZAADgAs6ePat9+/bpu+++048//qhTp04pKipKycnJateunekXfV+iDAAA8DMFBQX65JNPtGLFCkVERCguLk7x8fFKSEg4v6SvsqEMAAAs79ylcPny5Zo5c6YiIiLUu3dvXXXVVapZs6aCgoJMTli2KAMAAMs6t7Z/xYoVeuedd9SsWTPdfffduvLKKxUYGPiLuQCVGWUAAGBJR48e1bZt27RkyRKVlJTovvvuU/PmzS1TAH6OMgAAsJTMzEx9/vnn2rNnj0pKSpSSkqI2bdr4ZCc/f0UZAABYgsvl0ieffKKFCxcqPj5eiYmJatOmjYKDg82OZjrKAACgUjMMQ/n5+XrmmWd09uxZDR06VFdffbXCwsIq1fLAy8HZBACASqmkpES5ublatWqVXnnlFf3P//yPevfuff7wH/x/lAEAQKXjdruVlpamDz74QA0bNtSKFStKvU+/FXGbAABQqRw8eFBz585VcXGxOnTooKSkJOYF/AlGBgAAlYJhGFqxYoU++eQTde3aVYmJiapVq5bZsfwCZQAA4NcMw9Dhw4f19NNPq1q1ahozZowaNGhg6aWCF4vbBAAAv+V2u7V//36NHz9eCQkJGjNmTJkfG1wZUQYAAH4pKytLa9eu1erVq5WamqoOHTqYHclvUQYAAH5n9+7d+vjjjxUWFqYhQ4YoMjLS7Eh+jTkDAAC/4fV6lZaWpjlz5ui2225TYmKinE6n2bH8HmUAAOAX3G63Zs6cqQ0bNujFF19UeHg4kwR9hDIAAKjQvF6vjh8/rn//+98KDg7W5MmTJYlJgj5EGQAAVFhFRUX64osvtHbtWrVu3Vr9+vWjBJQBygAAoELyer36+OOPtWLFCt17771q06aNAgK4bJUFVhMAACqkV199Vfn5+Ro8eLAaNWpkdpxKjTIAAKgwDMNQUVGRnn/+ecXExOi//uu/WC1QDhhvAQBUCB6PRz/88INmzZqlli1bauDAgQoKCjI7liVQBgAApjMMQ5s2bdJbb72lIUOG6LrrrmPZYDniNgEAwHSfffaZVq9ereTkZCUlJZkdx3IoAwAA0xiGoYULF2rr1q26/fbbFRsba3YkS6IMAABM4Xa7tXjxYu3evVt33HGHIiMj2UPAJJQBAEC5MgxDbrdbCxYs0OHDh5WamqqoqCizY1kaEwgBAOVu0qRJOnPmjMaOHavq1aubHcfyGBkAAJSb4uJi/fWvf1WbNm3Ur18/ValSxexIEGUAAFAODMNQQUGBnnvuOXXt2lXXXXcdWwtXIJQBAECZMgxDp0+f1tSpU9WwYUMNGDBAdrvd7Fj4GWoZAKBMZWVlacqUKYqJidHAgQPNjoMLoJoBAMpMVlaW3nzzTUVHR2vEiBFmx8HvYGQAAFAmjh8/rkmTJqlbt27q3r272XHwBygDAACfMgxDJ0+e1DvvvKPk5GR17tyZzYQqOMoAAMBnzhWB2bNnq3Xr1urSpQtFwA9QBgAAPpORkaE5c+aoYcOGSklJMTsOSokJhAAAnzh16pQmTpyounXrasiQIWbHwUVgnwEAwGU7e/asXnjhBSUnJ6tHjx7cGvAz3CYAAFwywzBUVFSkN954Q507d1ZSUhJFwA9RBgAAl8zlcmnWrFmKjIxU7969KQJ+ijkDAIBL4vV6NX36dOXk5Gj06NEUAT/GyAAA4JJMmDBBNptN999/P2cN+DkmEAIALtrkyZNlt9s1YsQIhYaGmh0Hl4mRAQBAqXk8Hi1atEgej0e33XabnE6n2ZHgA4zrAABKxePx6IsvvtC+fft00003qVq1aswTqCQoAwCAP2UYhjZt2qT169erf//+qlOnjtmR4EOUAQDAn1qyZIkmTpyogQMHqkmTJmbHgY8xZwAA8LsMw9DBgwc1f/58PfXUU2revLnZkVAGWE0AALggwzCUnZ2tf/zjH7r99tsVHx/PHIFKipEBAMAFnT17VtOnT1dycrISEhLMjoMyxJwBAMBvuFwuzZ49WzVr1tR1111ndhyUMUYGAAC/8eabb8put2vQoEEKDg42Ow7KGGUAAHCeYRh67rnnlJ6erjfffFNVqlQxOxLKAWUAACDpp02F1q1bp8LCQk2ePJkiYCHMGQAAyOv1aufOnVq3bp3uvPNOVa1a1exIKEeUAQCAsrKytGDBAvXu3Vv169c3Ow7KGWUAACzO5XJp4sSJSkxM1NVXX212HJiAMgAAFubxePTMM8+odevW6tGjhxwOh9mRYALKAABYVHFxsZ544gkdPXpUgwcPVmBgoNmRYBLKAABYkNvt1qpVqxQREaHXXntNdjuXAyvjXx8ALGj79u3atGmThg0bprCwMLPjwGSlLgOzZ88uyxwAgHKSlZWluXPnqn///qpbt67ZcVABlLoMZGdn67333pPX6y3LPACAMlRcXKyXXnpJycnJatWqFacQQtJFlIERI0YoIyND69evl8fjKctMAIAykJubq/Hjxys+Pl7XX389KwdwXqnLQEREhG6++WZ99tln2r9/vwzDKMtcAAAfKi4u1rvvvqucnByNHDmSEQH8wkVNIGzevLk6deqkmTNnKi8vr6wyAQB8bPXq1SosLNQTTzzBygH8xkW/I7p166b4+HhNnDiR0QEA8APp6enauHGjBg8erGrVqpkdBxXQRZeBwMBADRo0SC6XSy+//LLcbndZ5AIAXCbDMHTy5EnNmzdPSUlJuvLKK7k9gAu6pLGigIAAPfXUU/r22281f/58VhgAQAVUXFys6dOnKzY2Vl27duX2AH7XJb8zgoOD9eKLL2r79u3asWOHLzMBAHxg2rRpcrlcGjVqFCMC+EOXVRNr166tvn37atmyZTp69KivMgEALtPMmTO1detW3X///WZHgR+4rDLgcDjUvn17xcXFacGCBSouLvZVLgDAJTAMQ5s3b9aePXv00EMPKTQ01OxI8AOXfQMpKChIgwcP1rFjx7RkyRLmDwCASQzD0LFjx7Ry5UrdcMMNatKkCbcHUCo+mU3icDj07LPP6v3339eXX37pi6cEAFwkt9utjz76SNHR0erUqRNFAKXm06mlzz//vN59911t3rzZl08LAPgThmFo0aJFys7O1tChQ82OAz/j0zLQqFEjjRo1SsuWLdOPP/7oy6cGAPyB1atX69tvv9WYMWMUEhJidhz4GZ+WAYfDoU6dOqlBgwZavny5CgsLffn0AIBfMQxD33zzjd544w3de++9ioyMNDsS/JDPd6AIDAzUrbfeql27dmnTpk1sWQwAZejUqVOaNm2annzyScXExJgdB36qTLajcjgceuSRRzR37lzt3LmzLF4CACyvoKBACxcuVGJiolq0aMGEQVyyMtubsk6dOrrzzjs1ZcoUZWRklNXLAIAleTweff755zp9+rR69uyp4OBgsyPBj5XpRtWtWrXSgAED9Mwzzyg7O7ssXwoALCUjI0MLFizQkCFDmCeAy2Yzyvimvsvl0qxZs5Sbm6uxY8dyUAYAXKaSkhL17NlTU6ZMUVxcnNlxUAmU+ZU5MDBQKSkpcrvdWrt2rTweT1m/JABUWrm5uXr00Uf16KOPKjY21uw4qCTKvAzYbDZFRUWpZ8+eWrt2rTIyMlhhAACXoKCgQFOmTFFISIi6dOnChEH4TLmN2bdq1UqdO3fWK6+8wvkFAHCRDMPQpk2blJOTo7Fjx3IAEXyqXG/gd+/eXa1atdJLL71Uni8LAH4vKytLy5Yt08CBAxUdHW12HFQy5VoGHA6HRowYoaKiIs2bN4/5AwBQCi6XS5MnT1bHjh3VsmVLs+OgEir3qf1BQUG66667NGvWLKWlpTF/AAD+gMfj0cyZMxUcHKwBAwbI4XCYHQmVULmXAZvNptq1a+uJJ57Ql19+qaysrPKOAAB+Y82aNdq0aZMee+wxJgyizJi26P+qq65SkyZNtGjRIg40AoALWL9+vd577z2NGzeOIoAyZVoZCAkJUUpKig4fPqz169dzuwAAfubo0aNasWKFBg0apPr161MGUKbKfAfCP1NUVKSePXvqgw8+UM2aNc2MAgAVgtvt1gcffKATJ07o3nvvVUBAgNmRUMmZvjdwSEiI3n77bT366KPMHwBgeYZhaMuWLdqwYYNSU1MpAigXppcBSWrcuLH69u2rf/3rX8rMzDQ7DgCYZt++fZo1a5buueceVa1a1ew4sIgKUQbsdrt69uypKlWq6NNPP2X/AQCWlJubq/Hjx+u2225Ts2bNzI4DC6kQZUCSrrjiCqWmpmr37t1KT09nQiEASzEMQxMnTlRycrLat29vdhxYTIUpA5IUExOjvn37atasWTp9+jSFAIAleDwezZgxQ0ePHlW3bt1YOYByV6HKgM1mU6dOndS4cWO9++67KikpMTsSAJQpwzCUnp6u9PR03X///YqKiqIMoNxVqDJwTmpqqk6dOqWPPvrI7CgAUKYKCws1f/58de3aVQkJCWbHgUVVyDIgSQ8//LC2bt2q9evXmx0FAMqEYRiaOnWqoqKilJycbHYcWFiFLQORkZHq16+fxo8fr++++475AwAqFcMw9Omnn2rfvn0aOXKkgoKCzI4EC6uwZcBms6ljx44aMmSI1q5dq6KiIrMjAYDP7N+/X//+97/1v//7vwoLCzM7DiyuwpYB6adC0KdPH+Xl5WndunXsPwCgUjh69Khee+01PfjggwoNDTU7DlCxy4D00/4Dt956qz799FPt3r3b7DgAcFnOnj2ruXPnKiYmRgkJCbLbK/yvYViAX7wLGzZsqNTUVD311FMqLi42Ow4AXBKv16sdO3YoMzNTI0eO1BVXXGF2JECSn5QBSYqPj9fIkSP1t7/9TV6v1+w4AHDR8vLy9Oqrr+quu+5SVFSU2XGA8/ymDNhsNt1www2Kjo7W7NmzGSEA4Ffy8/P1zDPPKDU1VQ0bNjQ7DvALflMGJMnpdColJUXr1q3Tt99+y3JDAH7B7XZr2rRpatiwoa6//np2GESF41dlQPrpuONbb71Vq1ev1unTp82OAwB/Ki0tTVlZWUpNTaUIoELyuzIgSZ06dVJ0dLTmzZvHckMAFdrWrVu1ePFi3XTTTXI6nWbHAS7IL8tAcHCwRowYwXbFACoswzB08uRJvf/++0pMTFSLFi0YFUCF5ZdlQPqpELz++uv6+9//rszMTLPjAMAvGIahL7/8UoGBgbrtttvkcDjMjgT8Lr8tA5LkcDj04osvatKkSTp27JjZcQDgvM2bN2vlypV64IEHGBFAhefXZcBms6l169Zq2rSpZsyYodzcXLMjAYAOHjyoqVOnasyYMewnAL/g12VA+ul2Qd++fXXy5Elt2bLF7DgALM7j8ej555/X8OHD1axZM7PjAKXi92VAksLDwzVy5EgtWrRIR44cYf8BAKYoKSnR9OnT1b59e7Vp04bbA/AblaIM2Gw2xcfHKykpSW+99ZYKCgrMjgTAYjwej9auXaudO3cqOTlZISEhZkcCSq1SlIFz+vfvr+rVq2vKlClmRwFgMadOndK8efOUkpKiBg0amB0HuCiVqgxI0t13363MzEytXLnS7CgALMLr9Wrq1Km69tprlZSUZHYc4KJVujLgdDp1++236/PPP9fu3buZPwCgTBmGofnz56uwsFBDhgxhngD8UqUrAzabTY0aNVLHjh01ffp0zi8AUKa+++47LV68WI899hjbDcNvVboyIP1UCLp37y6n06nVq1erpKTE7EgAKqGsrCy9/fbb+utf/6rg4GCz4wCXrFKWAUkKCwtTamqqtmzZoi1btnC7AIBP5eTkaN68eerRo4fi4uK4PQC/VmnLgCTFxMRoxIgRevnll9mdEIDPuN1uLVmyREePHlVycjKjAvB7lboMSFKzZs1055136tFHH5XX6zU7DgA/ZxiGTp8+rcWLF+uee+5ReHi42ZGAy1bpy4Akde7cWW3bttW0adPkdrvNjgPAj+Xn5+vvf/+7HnjgAdWtW9fsOIBPWKIMBAYGauDAgcrMzNSGDRsYIQBwSYqKivTmm2+qQ4cOSkxMZJ4AKg1LlAFJqlWrlpKSkrRgwQIdOXLE7DgA/NCSJUtUVFSk4cOHmx0F8CnLlAFJuvrqq3XVVVdp/vz5crlcZscB4Ee2bNmiXbt2afjw4QoICDA7DuBTlioDTqdTt956q7Kzs7Vy5UqWGwL4U4ZhKCsrS59++qk6d+6s2NhYbg+g0rFUGZCkkJAQPfvss3rttde0b98+s+MAqODcbrfee+89FRYWqnv37rLbLfdrExZgyXe1zWbTK6+8oldffVU//vij2XEAVGDffPONDh06pHHjxjEigErLsmWgcePGuu666/Thhx8qJyfH7EgAKqDvv/9ec+bM0dixY1W1alWz4wBlxpJlQJICAgKUnJyskpISffbZZyw3BPALZ8+e1YQJEzR69Gg1atTI7DhAmbJsGZB+Or+gZ8+emjlzpr7//nsmFAKQJHk8Hk2ePFk9e/ZUy5YtzY4DlDlLlwFJio+P14MPPshxxwAkSSUlJUpLS1NAQIC6d+8uh8NhdiSgzFm+DNhsNnXp0kWtWrXS66+/znHHgIUZhqH09HStWbNGPXv2VI0aNZg0CEuwfBk4Z9iwYZKkOXPmmJwEgFncbrfeeustdejQQQkJCWbHAcoNZeBn/vu//1t79+7Vl19+yfwBwGIMw9CkSZMUHx+vPn36mB0HKFeUgf+w2WyqVauW+vXrp1WrVunIkSMUAsAivF6vVq5cqcOHD+v2229XUFCQ2ZGAckUZ+Bmbzaarr75aderU0QcffKDCwkKzIwEoB7t379bChQv15JNPUgRgSZSBC0hJSdGhQ4e0atUqRgeASu748eP66KOPNHz4cFWvXt3sOIApKAMXEB0drXHjxiktLU3btm0zOw6AMlJYWKgVK1YoJiZGbdu25dwBWBbv/N9Rr149Pfzww3rqqad09uxZs+MA8DHDMLR69WqtW7dON954o0JDQ82OBJjGZjAO/rsMw9CaNWu0bNkyPfvsswoODjY7EgAfOXXqlIYMGaKZM2eqVq1aZscBTMXIwB+w2Wzq0KGDGjdurIULF6q4uNjsSAB8IDs7W88884z+7//+jyIAiDLwp8LCwtSnTx/t2bNHW7Zs4UAjwM/l5eXpvffeU2Jiojp06GB2HKBCoAyUQkxMjLp3767Jkydz3DHg55YsWaLAwED169ePCYPAf/CTUEodOnRQ//799dJLLzE6APghwzC0fft2paenq1+/fnI6nWZHAioMykApBQYGauDAgYqIiNDUqVM50AjwI4Zh6OjRo5o3b56Sk5MVFxfHAUTAz1AGLoLdbtcDDzyg9PR0rV692uw4AErJ6/XqueeeU5UqVdSlSxeKAPArlIGLFBgYqDFjxmjNmjXatWuX2XEAlMKcOXMUHh6uxx57zOwoQIVEGbhINptNMTExuu6667R8+XJlZ2ebHQnAH1i1apXS09M1btw4s6MAFRZl4BIEBASoc+fOstvtWrBggVwul9mRAPyKYRjas2ePPvvsM40YMULh4eFmRwIqLMrAJQoKCtKDDz6oNWvWaMOGDRxoBFQwubm5WrBggbp166bGjRszTwD4A5SBy2Cz2fTqq69q2rRpzB8AKhC3262lS5cqNDRU3bp1owgAf4IycJlq1qyp+++/X7NmzdL+/fvNjgNYnmEYmjVrltasWaMhQ4ZwpghQCpQBH2jZsqWSkpI0Z84cnTlzxuw4gKXt2bNHS5cu1SOPPMK5A0ApUQZ8wOFwKCkpSdWqVdPixYvZoRAwSX5+vsaNG6eJEyeqSZMmZscB/AZlwEcCAwN1yy23aOfOnVq/fj2FAChneXl5mjBhgh544AFFR0ebHQfwK5QBH7HZbKpZs6ZSUlL01ltvaefOnWZHAiyjqKhIy5cvV506ddSxY0c5HA6zIwF+hTLgY4mJiRo9erQmTJig3Nxcs+MAlZ7X69XWrVuVnp6u3r17q2rVqmZHAvyOzWCBvM8ZhqGVK1dqxYoVevnll1nWBJShwsJC3XLLLXrjjTcUFxdndhzALzEyUEaSkpLUokULTZs2jRMOgTJSVFSkQYMGaezYsYqNjTU7DuC3KANlwGazKSQkRH379tWJEyf01VdfyePxmB0LqFTOnDmjiRMnavTo0br++usZgQMuA2WgDNWuXVu9evXSqlWrdPDgQbYsBnykqKhIixcvVnh4uFJSUigCwGWiDJSxli1bqlOnTnr66acZHQB8wDAMbdiwQRkZGbr55psVEhJidiTA71EGykH37t3Vp08fPfHEE4wOAJfBMAxlZmZq4cKFuuWWW1SjRg2zIwGVAmWgHAQGBurmm29W/fr19e6778rtdpsdCfBLp0+f1gsvvKCbbrpJjRs3NjsOUGlQBspJQECAhg0bplOnTumzzz6jEAAX6cyZM3rkkUcUFRWlLl26ME8A8CH2GShnBw4c0IwZMzRo0CAlJCTwCw0oBZfLpXfeeUc2m01333237HY+xwC+xE9UOWvQoIF69eqlGTNmsEMhUEqLFy+Wy+XSyJEjKQJAGeCnygTt2rVTQkKC7rvvPlYYAH/AMAxt3rxZO3fu1M0336zQ0FCzIwGVEmXABA6HQyNGjFDTpk319NNPq6ioyOxIQIVjGIaOHDmijz76SCkpKYqJieG2GlBGKAMmsdvtevzxx1W1alV9/PHHKi4uNjsSUKEcP35cb7zxhjp27Ki2bdtSBIAyRBkwUUBAgO68804dOHBA69atYw8C4D8KCwv14osv6sorr1Tfvn3NjgNUepQBk4WHh2vw4MFau3atfvjhB7PjABXC66+/rlatWmnUqFFmRwEsgTJQAdSvX1833nijJk+erNOnT5sdBzCN1+vVhx9+qJCQEA0aNIiVA0A54SetArDb7br66qt1zTXX6JFHHtHJkyfNjgSUO6/Xq02bNik9PV0DBw5U1apVmScAlBPKQAVhs9k0dOhQNWnSRBMnTmQPAliKYRjau3evli9frj59+rByAChnlIEK5qGHHlKDBg00b948lhzCMg4dOqTx48frhhtuUOvWrc2OA1gOZaCCCQwM1JAhQ5Sfn6/ly5ezwgCVXkFBgR577DGNHDlS1157rdlxAEuiDFRAYWFhGjFihL744gtt27aNQoBKy+1269lnn9Xtt9+ujh07mh0HsCzKQAUVERGh++67T5MmTVJ6errZcQCfKyoq0uzZs9W0aVNOIQRMRhmooGw2m+rXr6/hw4fr+eef1+bNm82OBPhMSUmJPvnkE+Xm5iolJUUhISGUAcBEAWYHwB/r3Lmz8vLy9MEHH6h69epq2LCh2ZGAy7Z69Wrt2LFDo0aNUo0aNcyOA1iezeCGdIXn8Xi0bt06ffPNN0pNTVVkZCSfouCXDMPQsmXLNGXKFE2aNEm1atUyOxIAcZvALzgcDnXt2lVxcXFasGCB8vPzmVQIv+P1erVlyxbNnj1bb7zxhmrWrGl2JAD/QRnwEw6H4/ySw/nz58vr9ZodCSg1wzB04MABffTRR/rrX/+q6OhoRreACoQy4Gcefvhhff/995oxY4bZUYBSO3HihGbOnKlevXqpWbNmZscB8CuUAT/05JNP6vDhw3r77bfNjgL8KbfbrX/84x+69tprlZiYaHYcABdAGfBDVatW1ZgxY1RUVKQPP/yQ+QOosAzD0H333ac+ffooOTmZWwNABUUZ8EM2m00REREaMmSI9uzZo88//1wej8fsWMAvFBQUaPTo0WrcuLG6d+8uh8NhdiQAv4OlhX7uwIED5+/FtmvXjk9eqBByc3M1b948Va9eXQMGDFBQUJDZkQD8AUYG/FyDBg10yy23aNGiRfriiy/MjgOooKBAixYtUmhoqHr27EkRAPwAZaASaNasmYYOHarJkydr8eLFZseBhXm9Xs2ePVtut1t9+/bVFVdcYXYkAKVAGagkmjdvrscff1ybNm3ipEOYwuv1atq0acrLy9OQIUNUvXp1syMBKCXOJqgkbDabEhISZBiGli5dquDgYDVu3Fh2O30PZa+wsFBvvfWWdu7cqUmTJikwMNDsSAAuAleKSsRms6lly5bq1q2bFi1apL179zJCgDJXUFCgZcuWqbCwUP/85z8pAoAfYjVBJfXVV19p1apVuuGGG3TNNdeYHQeVlMvl0ooVK3T8+HH179+fg4cAP8XIQCV17bXXql+/fho/frzS0tLMjoNKyDAMffzxxzp48KAGDBhAEQD8GCMDlZhhGNq2bZs++OADDR8+XE2bNmUfAvhESUmJ3n//fR06dEgPPvigwsLCzI4E4DJQBio5wzC0ZcsWrVy5UgMHDlSTJk2YVIjLkp+fr9dff105OTl65pln2EcAqAS4KlRyNptNbdu2VVJSkj7++GNt377d7EjwY3l5efr444/l9Xr18MMPUwSASoKRAQvZsmWLli1bprZt26p3795mx4GfcblcmjVrliSpb9++qlmzpsmJAPgK+wxYSOvWreV0OvXPf/5ThmGoT58+ZkeCH3n55ZcVHR2tAQMGsKEQUMkwMmAxhmFo9+7deueddzRgwAB17tyZOQT4Q0VFRXr22WeVkJCgQYMGcWsAqIQoAxZkGIYOHTqkd999Vz169FC3bt0oBLig06dP67nnnlNCQoKGDx/OhkJAJUUZsLDMzEy9/fbbatGihW6++Waz46ACMQxDx44d08yZMxUeHq5bbrlF1apVMzsWgDJCGbC4nJwcvffeeyoqKtK9997LenFIkg4ePKhXXnlFvXr1UlJSErcGgEqOMmBxhmGooKBAc+fO1eHDh/XAAw+oWrVqbE5kUYZhaMeOHRo/frwee+wxNWvWjPcCYAGUAcgwDHk8Hs2bN08HDx7U7bffrqioKC4CFlNSUqL58+dr6dKlevHFF1W7dm3eA4BFUAbwCwsWLND27ds1bNgwNW7c2Ow4KCfFxcVKS0vTkiVLdM8996hly5ZmRwJQjigD+I1169Zp9erV6tKli5KTk82OgzLmcrk0depUFRYWauDAgYqLizM7EoByRhnAb5y7bzx9+nS1bdtWN910k0JCQsyOhTJw/PhxvfDCC2rXrp369OmjiIgIsyMBMAFlABfk9Xp15MgRTZ06VZGRkRo9ejSFoBIxDEPr16/Xu+++qzvvvFMdOnRgxQBgYZQB/K5zEwsnT56srKwsjRs3TuHh4WbHwmUwDENer1ebNm3Syy+/rDvvvFM9evRgoiBgcZQBlMqiRYu0bt063XbbbUpISFBwcLDZkXAJcnNzlZaWpq+++kqpqamKj483OxKACoAygFI592lyzpw5uvrqqzVgwABVqVLF7Fi4CHv37tWiRYvkcrl0xx13KDIy0uxIACoIygBKzev16uDBg1q8eLHy8vI0btw45hH4ibS0NC1fvlw9evRQr169OIsCwC/wGwGlZrfb1aBBA40ePVoJCQnq37+/9u3bJ6/Xa3Y0/A6Xy6VZs2ZpwoQJSklJ0Q033EARAPAbjAzgkpxbbfCXv/xF/fr104033qiwsDAmolUQXq9X2dnZeuutt1RcXKwnnnhCoaGh/PsAuCDKAC5LZmamXnrpJUVFRemmm25SkyZN+ORpslOnTmnjxo365JNP1KlTJw0aNMjsSAAqOMoALltWVpZWrlypffv2cRyyiQzDUFZWliZNmqT8/HyNGjVK8fHxlDMAf4oyAJ/54YcftHTpUqWnp+svf/kLZxuUs08++URTpkxR7969dcMNN6h27dpmRwLgJygD8JlzxyF/9dVXmjhxosaOHavu3bsrICCAe9VlxOv16syZM5o4caJycnL0+OOPKzw8nN0EAVwUygB86tzbadOmTXr++efVpk0bjRgxQnXr1uUC5UOGYejs2bP67LPP9M4772jw4MG69dZb+X8M4JJQBlCm5s+fr/Xr16tDhw7q2rWrYmJizI7k9zwejzZv3qx169Zp3759GjZsmDp16mR2LAB+jDKAMmUYhg4dOqQlS5Zo79696tSpk/r168d2xpfowIEDmjt3rlwul1q3bq3OnTtz0iCAy0YZQLkoKCjQjh07tGrVKqWnp2vs2LFq166d2bH8RmFhoaZOnaqNGzeqf//+uuaaa1SnTh3mYgDwCcoAyo1hGCoqKtLu3bs1YcIE1a1bV3fccYdiY2OZZHgBXq9XLpdLGzZs0IQJE1S/fn3dddddatKkiQICAsyOB6ASoQyg3J0rBQsWLNCMGTPUt29f9ezZU7GxsQoNDTU7num8Xq9OnDihXbt2aenSpcrPz9f999+vZs2aSRKlCYDPUQZgqu3bt+vTTz/V8ePHFRcXpzZt2qhly5aWPRExMzNTGzZs0Jdffqn8/HwNGjRInTt3Zo4FgDJFGUCFcOjQIX399dfatWuXTp06pZ49e6pXr15yOBxmRysXmZmZWrhwoTIyMlS7dm01adJEbdu2ZeMgAOWCMoAKw+PxKDs7W2lpadq0aZN27NihUaNGqW/fvqpWrZqkyjFE/vMfuf379+u1117TgQMHlJKSosTERMXGxqpq1aomJgRgNZQBVDgej0dut1snTpzQ22+/rY0bN6pp06a67777VLt2bYWEhPjlBLpzcyWKioq0detWvffeezp8+LB69eqlYcOGqUaNGkykBGAKygAqvMzMTL3++uv65ptv1KpVK7Vv317NmjVTeHi46tatW6GLwbktmo8dO6Zjx47p008/1caNGxUbG6uhQ4eqU6dOFTo/AGugDMBvuN1ubdy4UV9//bWysrKUlZWl5s2bq3nz5mrcuLHi4uLkdDrNjilJOnnypLZv364DBw4oOztbeXl5ys/PV7169dStWze1adPG7IgAcB5lAH7n3NK7LVu2nC8Fx44dU15enpxOp5KSktSqVSvFxMSUy6duwzDkdru1detWpaena+vWrSooKFB+fr4aN26stm3bqlGjRmrQoIGCg4O5DQCgwqEMwK+dO7DnzJkzOnHihObOnav8/Hzt379fubm5io+P1xVXXKEePXqoadOmql279i+W6ZX2wvzzHxOXy6Vdu3Zpz5492rVrl3bu3Kk9e/YoNjb2/Kf+evXqyel0qnr16kwGBFDhUQZQaRiGIY/HI8Mw5PV6lZubq++++06zZ8+Wy+XS0aNHdfLkSYWHh6u4uFgJCQmqWbOmqlSpoipVquj48eOKjo5WUFCQ3G633G63du7cqaCgIBUVFSkrK0vHjx9Xbm6u6tWrp/bt2yshIUEJCQlq0qSJgoKCZLfbZbfbZbPZGAEA4DcoA7AUl8ul48eP6+uvv1ZAQIA8Ho/y8/OVl5en3bt3KyIiQtWqVVNgYKACAgJ07NgxNWjQQA0bNlRUVJSioqJUvXr18xd8AKgMKAMAAFic3ewAAADAXJQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALA4ygAAABZHGQAAwOIoAwAAWBxlAAAAi6MMAABgcZQBAAAsjjIAAIDFUQYAALC4/weJ1hqyIXcPWwAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from IPython.display import display, clear_output\n", "import time\n", "\n", "# Initialize environment\n", "state, _ = env.reset()\n", "state = discretize_state(state)\n", "done = False\n", "total_reward = 0\n", "\n", "# Test the policy\n", "while not done:\n", " action = np.argmax(Q_table[state])\n", " next_state, reward, done, truncated, _ = env.step(action)\n", " state = discretize_state(next_state)\n", " total_reward += reward\n", "\n", " # Render frame\n", " frame = env.render()\n", " plt.imshow(frame)\n", " plt.axis(\"off\")\n", " display(plt.gcf())\n", " clear_output(wait=True)\n", " time.sleep(0.3) \n", "\n", "print(\"Total reward:\", total_reward)\n", "env.close()\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.9" } }, "nbformat": 4, "nbformat_minor": 2 }