Initial Commit

.gitignore

@@ -0,0 +1,163 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+/shelf/
+/workspace.xml
+*.npy

Qlearning_pole.py

@@ -0,0 +1,222 @@
+import random
+
+import numpy as np
+import gym
+import time
+from tqdm import tqdm
+import configparser
+class Qlearning:
+    ###########################################################################
+    #   START - __init__ function
+    ###########################################################################
+    # INPUTS:
+    # env - Cart Pole environment
+    # alpha - step size
+    # gamma - discount rate
+    # epsilon - parameter for epsilon-greedy approach
+    # numberEpisodes - total number of simulation episodes
+
+    # numberOfBins - this is a 4 dimensional list that defines the number of grid points
+    # for state discretization
+    # that is, this list contains number of bins for every state entry,
+    # we have 4 entries, that is,
+    # discretization for cart position, cart velocity, pole angle, and pole angular velocity
+
+    # lowerBounds - lower bounds (limits) for discretization, list with 4 entries:
+    # lower bounds on cart position, cart velocity, pole angle, and pole angular velocity
+
+    # upperBounds - upper bounds (limits) for discretization, list with 4 entries:
+    # upper bounds on cart position, cart velocity, pole angle, and pole angular velocity
+    def __init__(self, env = gym.make('CartPole-v1'), file='config.ini'):
+        self.env = env
+        self.load_values(file)
+
+
+
+    def load_values(self,file):
+        config = configparser.ConfigParser()
+        config.read(file)
+
+        cart_velocity_min = float(config['Parameters']['cart_velocity_min'])
+        cart_velocity_max = float(config['Parameters']['cart_velocity_max'])
+        pole_angle_velocity_min = float(config['Parameters']['pole_angle_velocity_min'])
+        pole_angle_velocity_max = float(config['Parameters']['pole_angle_velocity_max'])
+        number_of_bins_position = int(config['Parameters']['number_of_bins_position'])
+        number_of_bins_velocity = int(config['Parameters']['number_of_bins_velocity'])
+        number_of_bins_angle = int(config['Parameters']['number_of_bins_angle'])
+        number_of_bins_angle_velocity = int(config['Parameters']['number_of_bins_angle_velocity'])
+        self.action_number = self.env.action_space.n
+        self.alpha = float(config['Parameters']['alpha'])
+        self.gamma = float(config['Parameters']['gamma'])
+        self.epsilon = float(config['Parameters']['epsilon'])
+        self.numEpisodes = int(config['Parameters']['number_episodes'])
+
+        self.upperBounds = self.env.observation_space.high
+        self.lowerBounds = self.env.observation_space.low
+        self.upperBounds[1] = cart_velocity_max
+        self.upperBounds[3] = pole_angle_velocity_max
+        self.lowerBounds[1] = cart_velocity_min
+        self.lowerBounds[3] = pole_angle_velocity_min
+
+        self.batch_size = int(config['Parameters']['batch_size'])
+
+        self.rewardsEpisode = 0
+        self.sumRewardsEpisode = []
+
+        # Update the number of bins
+        self.num_bins = [number_of_bins_position, number_of_bins_velocity, number_of_bins_angle,
+                         number_of_bins_angle_velocity]
+
+        self.replayBuffer = []
+        self.Q = np.random.uniform(0, 1, size=(self.num_bins[0], self.num_bins[1], self.num_bins[2], self.num_bins[3], self.action_number))
+
+    # Observation space is not discrete so we make it discrete
+    def returnIndexState(self, state):
+        position = state[0]
+        velocity = state[1]
+        angle = state[2]
+        angularVelocity = state[3]
+
+        cartPositionBin = np.linspace(self.lowerBounds[0], self.upperBounds[0], self.num_bins[0])
+        cartVelocityBin = np.linspace(self.lowerBounds[1], self.upperBounds[1], self.num_bins[1])
+        cartAngleBin = np.linspace(self.lowerBounds[2], self.upperBounds[2], self.num_bins[2])
+        cartAngularVelocityBin = np.linspace(self.lowerBounds[3], self.upperBounds[3], self.num_bins[3])
+
+        indexPosition = np.maximum(np.digitize(position, cartPositionBin) - 1, 0)
+        indexVelocity = np.maximum(np.digitize(velocity, cartVelocityBin) - 1, 0)
+        indexAngle = np.maximum(np.digitize(angle, cartAngleBin) - 1, 0)
+        indexAngularVelocity = np.maximum(np.digitize(angularVelocity, cartAngularVelocityBin) - 1, 0)
+
+        return tuple([indexPosition, indexVelocity, indexAngle, indexAngularVelocity])
+
+    def selectAction(self, state, index):
+        # First 10% episodes will be random
+        if index < self.numEpisodes * 0.1:
+            return np.random.choice(self.action_number)
+
+        # We generate a random number to decide if we are exploring or not.
+        randomNumber = np.random.random()
+
+        # Decay starts at 55%
+        if index > self.numEpisodes * 0.6:
+            self.epsilon = 0.999 * self.epsilon
+
+        # If satisfied we are exploring
+        if randomNumber < self.epsilon:
+            return np.random.choice(self.action_number)
+
+        # Else we are being greedy
+        else:
+            return np.random.choice(np.where(
+                self.Q[self.returnIndexState(state)] == np.max(self.Q[self.returnIndexState(state)]))[0])
+
+    def train(self):
+        for indexEpisode in tqdm(range(self.numEpisodes)):#, miniters=1):
+        #for indexEpisode in range(self.numEpisodes):
+            rewardsEpisode = []
+            (stateS, _) = self.env.reset()
+            stateS = list(stateS)
+            #print(f'Simulating Episode {indexEpisode}')
+            terminalState = False
+            steps = 0
+            # Add a steps limiter to shorten training time
+            while not terminalState and steps < 2000:
+                steps += 1
+                stateSIndex = self.returnIndexState(stateS)
+                actionA = self.selectAction(stateS, indexEpisode)
+
+                (stateSprime, reward, terminalState, _, _) = self.env.step(actionA)
+                rewardsEpisode.append(reward)
+                stateSprime = list(stateSprime)
+
+                # Store the experience in the buffer
+                self.replayBuffer.append([stateS,actionA,reward,stateSprime,terminalState])
+
+                stateSprimeIndex = self.returnIndexState(stateSprime)
+
+                QmaxPrime = np.max(self.Q[stateSprimeIndex])
+                if not terminalState:
+                    error = reward + self.gamma * QmaxPrime - self.Q[stateSIndex + (actionA,)]
+                    self.Q[stateSIndex + (actionA,)] = self.Q[stateSIndex + (actionA,)] + self.alpha * error
+                else:
+                    error = reward - self.Q[stateSIndex + (actionA,)]
+                    self.Q[stateSIndex + (actionA,)] = self.Q[stateSIndex + (actionA,)] + self.alpha * error
+
+                stateS = stateSprime
+
+            if indexEpisode % 5 == 0:
+                self.updateQValues()
+            #print("Sum of rewards {}".format(np.sum(rewardsEpisode)))
+            self.sumRewardsEpisode.append(np.sum(rewardsEpisode))
+
+
+    def updateQValues(self):
+        if len(self.replayBuffer)<self.batch_size:
+            return
+
+        # Select a random batch of experiences
+        batch = random.sample(self.replayBuffer, self.batch_size)
+
+        for experience in batch:
+            state,action,reward,next_state,done = experience
+            stateIndex = self.returnIndexState(state)
+            actionIndex = action
+
+            if not done:
+                next_stateIndex = self.returnIndexState(next_state)
+                QmaxPrime = np.max(self.Q[next_stateIndex])
+                error = reward + self.gamma * QmaxPrime - self.Q[stateIndex + (actionIndex,)]
+            else:
+                error = reward - self.Q[stateIndex + (actionIndex,)]
+            self.Q[stateIndex + (actionIndex,)] += self.alpha * error
+
+    def simulateLearnedStrategy(self,env1 = gym.make("CartPole-v1"), render=False):
+        import gym
+        import time
+        # Choose this line if you want to see how it behaves
+        #env1 = gym.make("CartPole-v1", render_mode='human')
+        (currentState, _) = env1.reset()
+        if render:
+            env1.render()
+        timeSteps = 3000
+        steps = 0
+        # obtained rewards at every time step
+        obtainedRewards = []
+        terminated = False
+        truncated = False
+        while (not (terminated or truncated)) or steps < timeSteps:
+            steps+=1
+            #print(timeIndex)
+            # select greedy actions
+            actionInStateS = np.random.choice(np.where(self.Q[self.returnIndexState(currentState)] == np.max(
+                self.Q[self.returnIndexState(currentState)]))[0])
+            currentState, reward, terminated, truncated, info = env1.step(actionInStateS)
+            obtainedRewards.append(reward)
+            time.sleep(0.05)
+            if (terminated):
+                time.sleep(1)
+                break
+        return obtainedRewards, env1
+
+    def simulateRandomStrategy(self):
+        env2 = gym.make('CartPole-v1')
+        (currentState, _) = env2.reset()
+        #env2.render()
+        # number of simulation episodes
+        episodeNumber = 100
+        # time steps in every episode
+        timeSteps = 1000
+        # sum of rewards in each episode
+        rewardsEpisode = []
+
+
+        for timeIndex in range(timeSteps):
+            random_action = env2.action_space.sample()
+            observation, reward, terminated, truncated, info = env2.step(random_action)
+            rewardsEpisode.append(reward)
+            if (terminated):
+                break
+
+        return np.sum(rewardsEpisode), env2
+
+

config.ini

@@ -0,0 +1,14 @@
+[Parameters]
+cart_velocity_min = -5
+cart_velocity_max = 5
+pole_angle_velocity_min = -10
+pole_angle_velocity_max = 10
+number_of_bins_position = 50
+number_of_bins_velocity = 50
+number_of_bins_angle = 50
+number_of_bins_angle_velocity = 50
+alpha = 0.15
+gamma = 1
+epsilon = 0.25
+number_episodes = 20000
+batch_size = 32

main.py

@@ -0,0 +1,55 @@
+
+import gymnasium as gym
+import numpy
+import numpy as np
+from Qlearning_pole import Qlearning
+import os
+
+# Rendering the environment
+# env=gym.make('CartPole-v1',render_mode='human')
+
+
+Q1 = Qlearning()
+# run the Q-Learning algorithm
+Q1.train()
+# simulate the learned strategy
+(obtainedRewardsOptimal, env1) = Q1.simulateLearnedStrategy()
+
+# close the environment
+env1.close()
+# get the sum of rewards
+np.sum(obtainedRewardsOptimal)
+import matplotlib.pyplot as plt
+# now simulate a random strategy
+(obtainedRewardsRandom, env2) = Q1.simulateRandomStrategy()
+plt.figure(figsize=(12, 5))
+# plot the figure and adjust the plot parameters
+numpy.save("Qmatrix_new.npy",Q1.Q)
+plt.plot(Q1.sumRewardsEpisode, color='blue', linewidth=1)
+plt.xlabel('Episode')
+plt.ylabel('Reward')
+plt.yscale('log')
+plt.savefig('convergence.png')
+plt.title("Convergence of rewards")
+plt.show()
+
+
+# close the environment
+env1.close()
+# get the sum of rewards
+np.sum(obtainedRewardsOptimal)
+
+# now simulate a random strategy
+obtainedRewardsRandom = []
+for i in range(50):
+    (rewardsRandom, env2) = Q1.simulateRandomStrategy()
+    obtainedRewardsRandom.append(rewardsRandom)
+plt.title("Rewards with random strategy")
+plt.hist(obtainedRewardsRandom)
+plt.xlabel('Sum of rewards')
+plt.ylabel('Percentage')
+plt.savefig('histogram.png')
+plt.show()
+
+# run this several times and compare with a random learning strategy
+(obtainedRewardsOptimal, env1) = Q1.simulateLearnedStrategy()

readme.md

@@ -0,0 +1,63 @@
+# Cartpole Reinforcement Learning
+
+This repository is a project focused on exploring reinforcement learning techniques using the OpenAI Gym environment. The objective is to compare different algorithms and approaches to improve the performance of an agent in the Cartpole task.
+
+## Installation
+Installation of packages
+```
+pip install -r requirements.txt
+```
+
+If you want to execute the the training phase and get your own model execute the main program, the hyperparameters and different options can be changes via config.ini file.
+
+If you just want to watch the trained model play the game execute the following
+
+```
+python3 watchModel.py
+```
+
+
+## Objectives
+
+The main objectives of this project are as follows:
+
+1. Develop a working model that demonstrates an increase in survival time through training.
+2. Experiment with different reinforcement learning algorithms and compare their training time, complexity, and achieved scores.
+3. Fine-tune the algorithm parameters and the number of bins used to achieve optimal training results.
+4. Improve the consistency of the trained agent's strategy.
+5. Implement experience replay to enhance learning.
+
+## Results
+
+The initial approach used in this project was Q-Learning, and it produced the following results:
+
+![Convergence Plot](./resources/convergence_old.png)
+
+The convergence plot shows an increase in the score over time, with three distinct phases. The first phase corresponds to random inputs, followed by a phase where the model explores a lot. The third phase occurs when the epsilon value starts to decay.
+
+![Score Histogram](./resources/histogram_old.png)
+
+Comparing the results of the trained agent (after 20,000 episodes) with a random agent clearly demonstrates the improvement achieved:
+
+![Score Boxplot](./resources/boxplot_old.png)
+
+Despite the improvements, the trained agent still lacks consistency. This inconsistency is believed to be due to the inherent randomness in the Cartpole environment.
+
+## Experience Replay
+
+Experience replay has been implemented in this project, leading to significant improvements in the agent's performance. The details and results of this implementation are yet to be provided.
+
+The results of the trained agent with experience replay are as follows:
+It should be mention that to speed up the training phase, the experience replay agent had a score limit of 2000.
+| Metric                 | Old Agent | Trained Agent with Experience Replay |
+|------------------------|--------------|--------------------------------------|
+| Convergence Plot       | ![Convergence Plot](./resources/convergence_old.png) | ![Convergence Plot](./resources/convergence20kbuffer.png) |
+| Score Histogram        | ![Score Histogram](./resources/old_agent20k.png) | ![Score Histogram](./resources/trained_agent20k.png) |
+|Boxplot|![Score Boxplot](./resources/old_boxplot.png)| ![Score Boxplot](./resources/boxplot.png)|
+
+As observed by adding experience replay the agent has been able to objectively increase it's score.
+
+## References
+
+- https://arxiv.org/pdf/2112.04213.pdf
+- https://aleksandarhaber.com/q-learning-in-python-with-tests-in-cart-pole-openai-gym-environment-reinforcement-learning-tutorial/

requirements.txt

@@ -0,0 +1,6 @@
+gym==0.26.2
+gymnasium==0.28.1
+matplotlib==3.7.1
+numba==0.57.0
+numpy==1.23.5
+tqdm==4.65.0

test_weights.py

@@ -0,0 +1,40 @@
+import numpy as np
+from Qlearning_pole import Qlearning
+from tqdm import tqdm
+import matplotlib.pyplot as plt
+
+
+# Number of games the agent will play.
+iterations = 50
+agent = Qlearning()
+# Insert the weights of the Agent to plot.
+agent.Q = np.load("Qmatrix.npy")
+
+scores = []
+for i in tqdm(range(iterations),miniters=1,desc="Trained Agent"):
+    a, b = agent.simulateLearnedStrategy()
+    scores.append(np.sum(a))
+
+random_scores = []
+for i in tqdm(range(iterations),miniters=1,desc="Random Agent"):
+    a, b = agent.simulateRandomStrategy()
+    random_scores.append(a)
+
+data = [random_scores,scores]
+print(data)
+
+plt.title("Rewards with trained agent")
+plt.hist(scores)
+plt.xlabel('Reward')
+plt.ylabel('Percentage')
+# plt.savefig('./resources/new.png')
+plt.show()
+
+fig = plt.figure(figsize=(10,7))
+ax = fig.add_subplot(111)
+
+
+bp = ax.boxplot(data,patch_artist=True,notch=True,vert=0)
+plt.title("Trained Agent vs Random Agent (50 episodes)")
+# plt.savefig("./resources/old_boxplot.png")
+plt.show()

time.py

@@ -0,0 +1,32 @@
+from numba import jit, cuda
+import numpy as np
+# to measure exec time
+from tqdm import tqdm
+from timeit import default_timer as timer
+
+
+# normal function to run on cpu
+def func(a):
+    for i in range(100000000):
+        a[i] += 1
+        if i % 100 == 0:
+            print(i)
+    # function optimized to run on gpu
+
+
+def func2(a):
+    for i in tqdm(range(100000000)):
+        a[i] += 1
+
+
+if __name__ == "__main__":
+    n = 100000000
+    a = np.ones(n, dtype=np.float64)
+    """
+    start = timer()
+    func(a)
+    print("Prints", timer() - start)
+"""
+    start = timer()
+    func2(a)
+    print("tqdm", timer() - start)

watchModel.py

@@ -0,0 +1,8 @@
+import gymnasium as gym
+from Qlearning_pole import Qlearning
+import numpy as np
+if __name__ == '__main__':
+    env = gym.make('CartPole-v1',render_mode='human')
+    q = Qlearning(env)
+    q.Q = np.load('Qmatrix.npy')
+    q.simulateLearnedStrategy(render=True,env1=env)