DQN_v1 converging

DQN_v1.ipynb

@@ -13,20 +13,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "DDf1gLC2NTiK"
-      },
-      "outputs": [],
-      "source": [
-        "# !pip install -r ./requirements.txt\n",
-        "!pip install stable_baselines3[extra]\n",
-        "!pip install huggingface_sb3\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 2,
+      "execution_count": 1,
       "metadata": {
         "id": "LNXxxKojNTiL"
       },
@@ -35,7 +22,7 @@
           "name": "stderr",
           "output_type": "stream",
           "text": [
-            "2022-12-21 23:28:04.436066: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
+            "2022-12-22 18:43:04.111595: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
             "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
             "\n"
           ]
@@ -55,6 +42,7 @@
         "import numpy as np\n",
         "import random\n",
         "from matplotlib import pyplot as plt\n",
+        "from sklearn.preprocessing import MinMaxScaler\n",
         "\n",
         "import io\n",
         "import base64\n",
@@ -63,7 +51,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 16,
+      "execution_count": 29,
       "metadata": {},
       "outputs": [],
       "source": [
@@ -76,11 +64,11 @@
         "        # Hyperparameters\n",
         "        self.gamma = 0.95           # Discount rate\n",
         "        self.epsilon = 1.0          # Exploration rate\n",
-        "        self.epsilon_min = 0.05      # Minimal exploration rate (epsilon-greedy)\n",
-        "        self.epsilon_decay = 0.90    # Decay rate for epsilon\n",
-        "        self.update_rate = 200       # Number of steps until updating the target network\n",
+        "        self.epsilon_min = 0.001      # Minimal exploration rate (epsilon-greedy)\n",
+        "        self.epsilon_decay = 0.95    # Decay rate for epsilon\n",
+        "        self.update_rate = 5       # Number of steps until updating the target network\n",
         "        self.batch_size = 100\n",
-        "        self.learning_rate = 0.001\n",
+        "        self.learning_rate = 2.5e-4\n",
         "        \n",
         "        # Construct DQN models\n",
         "        self.model = self._build_model()\n",
@@ -90,120 +78,116 @@
         "        self.env = env\n",
         "        self.action_size = action_size\n",
         "\n",
+        "        self.scaler = None\n",
+        "\n",
         "    def _build_model(self):\n",
         "        model = tf.keras.Sequential()\n",
         "        \n",
         "        model.add(tf.keras.Input(shape=(4,)))\n",
-        "        # FC Layers\n",
-        "        model.add(layers.Dense(24, activation='relu'))\n",
-        "        model.add(layers.Dense(24, activation='relu'))\n",
-        "        model.add(layers.Dense(self.action_size, activation='linear'))\n",
+        "        model.add(layers.Dense(512, activation = 'relu'))\n",
+        "        model.add(layers.Dense(256, activation = 'relu'))\n",
+        "        model.add(layers.Dense(128, activation = 'relu'))\n",
+        "        model.add(layers.Dense(self.action_size, activation = 'linear'))\n",
+        "        # model.compile(optimizer = RMSprop(lr = self.lr, rho = 0.95, epsilon = 0.01), loss = \"mse\", metrics = ['accuracy'])\n",
         "        \n",
         "        optimizer = tf.keras.optimizers.Adam(learning_rate=self.learning_rate)\n",
-        "        model.compile(loss='mse', optimizer=optimizer, metrics=['mse'])\n",
+        "        # model.compile(loss='mse', optimizer=tf.keras.optimizers.RMSprop(lr = self.learning_rate, rho = 0.95, epsilon = 0.01), metrics = ['accuracy'])\n",
+        "        model.compile(loss='mse', optimizer=optimizer, metrics = ['accuracy'])\n",
         "        return model\n",
         "\n",
+        "    def _min_max(self):\n",
+        "        \"\"\"Run some steps to get data to do MINMAX scale \"\"\"\n",
+        "        state_arr = []\n",
+        "        state = self.env.reset()\n",
+        "        state_arr.append(self.env.observation_space.high)\n",
+        "        state_arr.append(self.env.observation_space.low)\n",
+        "        for i in range(1000):\n",
+        "            random_action = self.env.action_space.sample()\n",
+        "            next_state, reward, done, info = self.env.step(random_action)\n",
+        "            state_arr.append(next_state)\n",
+        "            if done:\n",
+        "                state = self.env.reset()\n",
+        "\n",
+        "        state_arr = np.array(state_arr)\n",
+        "        self.scaler = MinMaxScaler()\n",
+        "        self.scaler.fit(state_arr)\n",
         "\n",
         "    #\n",
         "    # Trains the model using randomly selected experiences in the replay memory\n",
         "    #\n",
         "    def _train(self):\n",
-        "        minibatch = random.sample(self.replay_buffer, self.batch_size)\n",
-        "        \n",
-        "        for state, action, reward, next_state, done in minibatch:\n",
-        "            \n",
-        "            if not done:\n",
-        "                model_predict = self.model.predict(np.array([next_state]), verbose=0)\n",
-        "                max_action = np.argmax(model_predict[0])\n",
-        "                target = (reward + self.gamma * self.target_model.predict(np.array([next_state]), verbose=0)[0][max_action])\n",
-        "            else:\n",
-        "                target = reward\n",
-        "                \n",
-        "            # Construct the target vector as follows:\n",
-        "            # 1. Use the current model to output the Q-value predictions\n",
-        "            target_f = self.model.predict(np.array([state]), verbose=0)\n",
-        "            \n",
-        "            # 2. Rewrite the chosen action value with the computed target\n",
-        "            target_f[0][action] = target\n",
-        "            \n",
-        "            # 3. Use vectors in the objective computation\n",
-        "            history = self.model.fit(np.array([state]), target_f, epochs=1, verbose=0)\n",
-        "            print(f\"Loss: {history.history['loss']}  \")\n",
-        "            \n",
-        "        if self.epsilon > self.epsilon_min:\n",
-        "            self.epsilon *= self.epsilon_decay\n",
-        "    #\n",
-        "    # Trains the model using randomly selected experiences in the replay memory\n",
-        "    #\n",
-        "    def _train_b(self):\n",
-        "        \n",
+        "        X, y = [], []\n",
         "        # state, action, reward, next_state, done \n",
         "        # create the targets \n",
-        "        mb_arr = np.array(random.sample(self.replay_buffer, self.batch_size), dtype=object)\n",
+        "        if self.batch_size > len(self.replay_buffer):\n",
+        "            return\n",
+        "        minibatch = random.sample(self.replay_buffer, self.batch_size)\n",
+        "        mb_arr = np.array(minibatch, dtype=object)\n",
         "\n",
         "        next_state_arr = np.stack(mb_arr[:,3])\n",
-        "        target_model_predict = self.target_model.predict(next_state_arr, verbose=0)\n",
-        "        max_action_arr = np.argmax(target_model_predict, axis=1)\n",
-        "        q_targets = []\n",
-        "        for idx,val in enumerate(zip(target_model_predict, max_action_arr)):\n",
-        "            row, col = val\n",
-        "            # if done\n",
-        "            if mb_arr[idx,4] == True:\n",
-        "                q_targets.append(mb_arr[idx,2])\n",
+        "        future_qvalues = self.target_model.predict(next_state_arr, verbose=0)\n",
+        "\n",
+        "        state_arr = np.stack(mb_arr[:,0])\n",
+        "        qvalues = self.model.predict(state_arr, verbose=0)\n",
+        "\n",
+        "        for index, (state, action, reward, next_state, done) in enumerate(minibatch):\n",
+        "            if done == True:\n",
+        "                q_target = reward\n",
         "            else:\n",
-        "                q_targets.append(row[col])\n",
+        "                q_target = reward + self.gamma * np.max(future_qvalues[index])\n",
         "\n",
-        "        q_targets = np.array(q_targets)\n",
-        "        reward_arr = np.stack(mb_arr[:,2])\n",
-        "        # targets Yj\n",
-        "        target_arr = (reward_arr + self.gamma * q_targets)\n",
+        "            q_curr = qvalues[index]\n",
+        "            q_curr[action] = q_target                \n",
+        "            X.append(state)\n",
+        "            y.append(q_curr)\n",
         "\n",
         "        # Perform gradient step\n",
-        "        state_arr = np.stack(mb_arr[:,0])\n",
-        "        model_predict = self.model.predict(state_arr, verbose=0)\n",
-        "        action_arr = np.stack(mb_arr[:,1])\n",
-        "        f_targets=[]\n",
-        "        for idx, val in enumerate(zip(action_arr, target_arr)):\n",
-        "            act, targ = val\n",
-        "            model_predict[idx][act] = targ\n",
+        "        X, y = np.array(X), np.array(y)\n",
+        "        history = self.model.fit(X, y, batch_size = self.batch_size, shuffle = False, verbose=0)\n",
+        "        # history = self.model.fit(X, y, epochs=1, verbose=0)\n",
+        "        # print(f\"Loss: {history.history['loss']}  \")\n",
         "\n",
-        "        history = self.model.fit(state_arr, model_predict, epochs=1, verbose=0)\n",
-        "        print(f\"Loss: {history.history['loss']}  \")\n",
-        "        # update epsilon\n",
-        "        if self.epsilon > self.epsilon_min:\n",
-        "            self.epsilon *= self.epsilon_decay\n",
         "\n",
         "    def learn(self, total_steps=None):\n",
-        "\n",
-        "        state = self.env.reset()\n",
+        "        #create scaler\n",
+        "        self._min_max()\n",
+        "        current_episode = 0\n",
         "        total_reward = 0\n",
-        "        rewards = []\n",
-        "        for current_step in tqdm(range(total_steps)):\n",
-        "\n",
-        "            # e-greedy\n",
-        "            if np.random.rand() <= self.epsilon:\n",
-        "                action = random.randrange(self.action_size)\n",
-        "            else:\n",
-        "                model_predict = self.model.predict(np.array([state]), verbose=0)\n",
-        "                action = np.argmax(model_predict[0])\n",
+        "        rewards = [0]\n",
+        "        current_step = 0\n",
+        "        while current_step < total_steps:\n",
+        "            current_episode += 1\n",
+        "            state = self.env.reset()\n",
+        "            total_reward = 0\n",
+        "            done = False\n",
+        "            while done != True:\n",
+        "                current_step +=1\n",
+        "                # e-greedy\n",
+        "                if np.random.random() > (1 - self.epsilon):\n",
+        "                    action = random.randrange(self.action_size)\n",
+        "                else:\n",
+        "                    model_predict = self.model.predict(np.array([state]), verbose=0)\n",
+        "                    action = np.argmax(model_predict)\n",
         "\n",
-        "            # step\n",
-        "            next_state, reward, done, info = self.env.step(action)\n",
-        "            total_reward += reward\n",
-        "            # add to buffer\n",
-        "            self.replay_buffer.append((state, action, reward, next_state, done))\n",
+        "                # step\n",
+        "                next_state, reward, done, info = self.env.step(action)\n",
+        "                total_reward += reward\n",
         "\n",
-        "            if done:\n",
-        "                rewards.append(total_reward)\n",
-        "                total_reward = 0\n",
-        "                state = self.env.reset()\n",
+        "                # add to buffer\n",
+        "                self.replay_buffer.append((state, action, reward, next_state, done))\n",
         "\n",
-        "            if current_step>10 and current_step % self.update_rate == 0:\n",
-        "                print(f\"epsilon:{self.epsilon} step:{current_step}  mean_reward {np.mean(rewards)} \")\n",
-        "                self._train()\n",
-        "                # update target\n",
-        "                self.target_model.set_weights(self.model.get_weights())\n",
+        "                if current_step>10 and current_step % self.update_rate == 0:\n",
+        "                    print(f\"epsilon:{self.epsilon} step:{current_step} episode:{current_episode} last_score {rewards[-1]} \")\n",
+        "                    self._train()\n",
+        "                    # update target\n",
+        "                    self.target_model.set_weights(self.model.get_weights())\n",
+        "                \n",
+        "                state = next_state\n",
         "            \n",
+        "            rewards.append(total_reward)\n",
+        "            # update epsilon\n",
+        "            if self.epsilon > self.epsilon_min:\n",
+        "                self.epsilon *= self.epsilon_decay\n",
         "    #\n",
         "    # Loads a saved model\n",
         "    #\n",
@@ -229,89 +213,73 @@
         "env = gym.make('CartPole-v1')\n",
         "\n",
         "model = DQN(env=env, replay_buffer_size=10_000, action_size=2)\n",
-        "model.learn(total_steps=20_000)\n",
+        "model.learn(total_steps=6_000)\n",
         "env.close()"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "# env = gym.make('CartPole-v1')\n",
-        "\n",
-        "# model = DQN(env=env, replay_buffer_size=10_000, action_size=2)\n",
-        "\n",
-        "# state = model.env.reset()\n",
-        "# for i in range(100):\n",
-        "#     random_action = env.action_space.sample()\n",
-        "#     next_state, reward, done, info = model.env.step(random_action)\n",
-        "#     model.replay_buffer.append((state, random_action, reward, next_state, done))\n",
-        "#     if done:\n",
-        "#         state = model.env.reset()\n",
-        "#     else:\n",
-        "#         state = next_state\n",
-        "\n",
-        "# minibatch = random.sample(model.replay_buffer, 10)\n",
-        "# mb = np.array(minibatch, dtype=object)\n",
-        "# print(mb[:,0])\n",
-        "# np.stack(mb[:,0])\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 6,
+      "execution_count": 31,
       "metadata": {},
       "outputs": [],
       "source": [
-        "model.save(\"./m1.h5\")"
+        "model.save(\"./alt/m1.h5\")"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 7,
+      "execution_count": 33,
       "metadata": {},
       "outputs": [
         {
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "Model: \"sequential_2\"\n",
+            "Model: \"sequential_28\"\n",
             "_________________________________________________________________\n",
             " Layer (type)                Output Shape              Param #   \n",
             "=================================================================\n",
-            " dense_6 (Dense)             (None, 128)               640       \n",
+            " dense_97 (Dense)            (None, 512)               2560      \n",
             "                                                                 \n",
-            " dense_7 (Dense)             (None, 64)                8256      \n",
+            " dense_98 (Dense)            (None, 256)               131328    \n",
             "                                                                 \n",
-            " dense_8 (Dense)             (None, 2)                 130       \n",
+            " dense_99 (Dense)            (None, 128)               32896     \n",
+            "                                                                 \n",
+            " dense_100 (Dense)           (None, 2)                 258       \n",
             "                                                                 \n",
             "=================================================================\n",
-            "Total params: 9,026\n",
-            "Trainable params: 9,026\n",
+            "Total params: 167,042\n",
+            "Trainable params: 167,042\n",
             "Non-trainable params: 0\n",
             "_________________________________________________________________\n",
-            "1.0 {}\n"
+            "Total reward 500.0\n"
           ]
         }
       ],
       "source": [
         "eval_env = gym.make('CartPole-v1')\n",
         "model = DQN(env=eval_env, replay_buffer_size=10_000, action_size=2)\n",
-        "model.load(\"./m1.h5\")\n",
+        "model.load(\"./alt/m1.h5\")\n",
         "eval_env = wrappers.Monitor(eval_env, \"./alt/gym-results\", force=True)\n",
         "state = eval_env.reset()\n",
+        "total_reward = 0\n",
         "for _ in range(1000):\n",
         "    action = model.play(state)\n",
         "    observation, reward, done, info = eval_env.step(action)\n",
-        "    # print(info)\n",
+        "    total_reward +=reward\n",
         "    state = observation\n",
         "    if done: \n",
-        "        print(reward, info)\n",
+        "        print(f\"Total reward {total_reward}\")\n",
         "        break\n",
         "eval_env.close()"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
     }
   ],
   "metadata": {