Initial commit

README.md

@@ -6,7 +6,7 @@ tags:
 - reinforcement-learning
 - stable-baselines3
 model-index:
-- name: DQN
+- name: PPO
   results:
   - task:
       type: reinforcement-learning
@@ -16,13 +16,13 @@ model-index:
       type: SpaceInvadersNoFrameskip-v4
     metrics:
     - type: mean_reward
-      value: 374.00 +/- 214.89
+      value: 808.50 +/- 361.36
       name: mean_reward
       verified: false
 ---
 
-# **DQN** Agent playing **SpaceInvadersNoFrameskip-v4**
-This is a trained model of a **DQN** agent playing **SpaceInvadersNoFrameskip-v4**
+# **PPO** Agent playing **SpaceInvadersNoFrameskip-v4**
+This is a trained model of a **PPO** agent playing **SpaceInvadersNoFrameskip-v4**
 using the [stable-baselines3 library](https://github.com/DLR-RM/stable-baselines3)
 and the [RL Zoo](https://github.com/DLR-RM/rl-baselines3-zoo).
 
@@ -38,39 +38,37 @@ SB3 Contrib: https://github.com/Stable-Baselines-Team/stable-baselines3-contrib
 
 ```
 # Download model and save it into the logs/ folder
-python -m rl_zoo3.load_from_hub --algo dqn --env SpaceInvadersNoFrameskip-v4 -orga Roberto -f logs/
-python enjoy.py --algo dqn --env SpaceInvadersNoFrameskip-v4  -f logs/
+python -m rl_zoo3.load_from_hub --algo ppo --env SpaceInvadersNoFrameskip-v4 -orga Roberto -f logs/
+python enjoy.py --algo ppo --env SpaceInvadersNoFrameskip-v4  -f logs/
 ```
 
 If you installed the RL Zoo3 via pip (`pip install rl_zoo3`), from anywhere you can do:
 ```
-python -m rl_zoo3.load_from_hub --algo dqn --env SpaceInvadersNoFrameskip-v4 -orga Roberto -f logs/
-rl_zoo3 enjoy --algo dqn --env SpaceInvadersNoFrameskip-v4  -f logs/
+python -m rl_zoo3.load_from_hub --algo ppo --env SpaceInvadersNoFrameskip-v4 -orga Roberto -f logs/
+rl_zoo3 enjoy --algo ppo --env SpaceInvadersNoFrameskip-v4  -f logs/
 ```
 
 ## Training (with the RL Zoo)
 ```
-python train.py --algo dqn --env SpaceInvadersNoFrameskip-v4 -f logs/
+python train.py --algo ppo --env SpaceInvadersNoFrameskip-v4 -f logs/
 # Upload the model and generate video (when possible)
-python -m rl_zoo3.push_to_hub --algo dqn --env SpaceInvadersNoFrameskip-v4 -f logs/ -orga Roberto
+python -m rl_zoo3.push_to_hub --algo ppo --env SpaceInvadersNoFrameskip-v4 -f logs/ -orga Roberto
 ```
 
 ## Hyperparameters
 ```python
-OrderedDict([('batch_size', 32),
-             ('buffer_size', 100000),
+OrderedDict([('batch_size', 256),
+             ('clip_range', 'lin_0.1'),
+             ('ent_coef', 0.01),
              ('env_wrapper',
               ['stable_baselines3.common.atari_wrappers.AtariWrapper']),
-             ('exploration_final_eps', 0.01),
-             ('exploration_fraction', 0.1),
              ('frame_stack', 4),
-             ('gradient_steps', 1),
-             ('learning_rate', 0.0001),
-             ('learning_starts', 100000),
-             ('n_timesteps', 100000.0),
-             ('optimize_memory_usage', False),
+             ('learning_rate', 'lin_2.5e-4'),
+             ('n_envs', 8),
+             ('n_epochs', 4),
+             ('n_steps', 128),
+             ('n_timesteps', 10000000.0),
              ('policy', 'CnnPolicy'),
-             ('target_update_interval', 1000),
-             ('train_freq', 4),
+             ('vf_coef', 0.5),
              ('normalize', False)])
 ```

args.yml

@@ -1,6 +1,6 @@
 !!python/object/apply:collections.OrderedDict
 - - - algo
-    - dqn
+    - ppo
   - - conf_file
     - null
   - - device
@@ -54,7 +54,7 @@
   - - save_replay_buffer
     - false
   - - seed
-    - 976994039
+    - 1410119825
   - - storage
     - null
   - - study_name

config.yml

@@ -1,29 +1,25 @@
 !!python/object/apply:collections.OrderedDict
 - - - batch_size
-    - 32
-  - - buffer_size
-    - 100000
+    - 256
+  - - clip_range
+    - lin_0.1
+  - - ent_coef
+    - 0.01
   - - env_wrapper
     - - stable_baselines3.common.atari_wrappers.AtariWrapper
-  - - exploration_final_eps
-    - 0.01
-  - - exploration_fraction
-    - 0.1
   - - frame_stack
     - 4
-  - - gradient_steps
-    - 1
   - - learning_rate
-    - 0.0001
-  - - learning_starts
-    - 100000
+    - lin_2.5e-4
+  - - n_envs
+    - 8
+  - - n_epochs
+    - 4
+  - - n_steps
+    - 128
   - - n_timesteps
-    - 100000.0
-  - - optimize_memory_usage
-    - false
+    - 10000000.0
   - - policy
     - CnnPolicy
-  - - target_update_interval
-    - 1000
-  - - train_freq
-    - 4
+  - - vf_coef
+    - 0.5

ppo-SpaceInvadersNoFrameskip-v4.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b655fac000369f7b2d10caa6a6ee777aed467e6a90a635ae0d5224043d5638db
+size 20442626

ppo-SpaceInvadersNoFrameskip-v4/_stable_baselines3_version

@@ -0,0 +1 @@
+1.6.2

ppo-SpaceInvadersNoFrameskip-v4/policy.optimizer.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:223a39a5f8f477e61ba0ead2052ca913cb226730beacc7ea10af6408609e1cf5
+size 13511481

ppo-SpaceInvadersNoFrameskip-v4/policy.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39250ac8c632db02b9996e3db6398cb748e703da9faea8c3b183aa97d1c40be5
+size 6755009

ppo-SpaceInvadersNoFrameskip-v4/pytorch_variables.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d030ad8db708280fcae77d87e973102039acd23a11bdecc3db8eb6c0ac940ee1
+size 431

ppo-SpaceInvadersNoFrameskip-v4/system_info.txt

@@ -0,0 +1,7 @@
+OS: Linux-5.15.0-56-generic-x86_64-with-glibc2.35 #62~20.04.1-Ubuntu SMP Tue Nov 22 21:24:20 UTC 2022
+Python: 3.9.15
+Stable-Baselines3: 1.6.2
+PyTorch: 1.13.1+cu117
+GPU Enabled: True
+Numpy: 1.24.0
+Gym: 0.21.0

replay.mp4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2bdab282bd1eb89b9d6e06e4029b8262d97a8298038199be2dd79dd558b176c4
-size 234682
+oid sha256:85bfc6ed9433811788324a541ce8b129a920cb965bae7796fe02c1a772a65659
+size 203960

results.json

@@ -1 +1 @@
-{"mean_reward": 374.0, "std_reward": 214.89299662855464, "is_deterministic": false, "n_eval_episodes": 10, "eval_datetime": "2022-12-20T20:52:47.885895"}
+{"mean_reward": 808.5, "std_reward": 361.35889362239305, "is_deterministic": false, "n_eval_episodes": 10, "eval_datetime": "2022-12-21T09:29:07.019988"}

train_eval_metrics.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a25f47fcc3bae5627ec7a41f3b4886bbb8a8c40c5497a0805663ebddbe9d9dc6
-size 5763
+oid sha256:271a39b9269e2be53529507d07f422263036bf2d60154c534c274b85d139e3f2
+size 314162