Upload folder using huggingface_hub

checkpoints/checkpoint-pt-17500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:973c5df01a7e4998cac279fa8d88512468c87607d6d404d2ce7a7e82ae975b5f
+size 3550041880

checkpoints/checkpoint-pt-17500/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3b2ce7e08081f282216ae6d35b3e3e08ec13a6874529efce924280e2d044c09
+size 14344

checkpoints/checkpoint-pt-20000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31dc4320c0d516e13d139a4cd4bffc116827b038b463f8423675c3d60c450a38
+size 3550041880

checkpoints/checkpoint-pt-20000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3b2ce7e08081f282216ae6d35b3e3e08ec13a6874529efce924280e2d044c09
+size 14344

checkpoints/checkpoint-pt-20001/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31dc4320c0d516e13d139a4cd4bffc116827b038b463f8423675c3d60c450a38
+size 3550041880

checkpoints/checkpoint-pt-20001/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2add1dde78776714849e4d0f42e83551a9123fb6ef7d38a53150e7fc6c0d4124
+size 14344

checkpoints/main.log

@@ -677,3 +677,207 @@ Mixed precision type: bf16
 [2024-08-31 17:07:58,425][Main][INFO] - [train] Step 15500 out of 20000 | Loss --> 1.773 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.378 | Lr --> 0.002 | Seconds_per_step --> 4.876 | 
 [2024-08-31 17:10:00,358][Main][INFO] - [train] Step 15525 out of 20000 | Loss --> 1.778 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.364 | Lr --> 0.002 | Seconds_per_step --> 4.877 | 
 [2024-08-31 17:12:03,531][Main][INFO] - [train] Step 15550 out of 20000 | Loss --> 1.773 | Grad_l2 --> 0.200 | Weights_l2 --> 11272.349 | Lr --> 0.002 | Seconds_per_step --> 4.927 | 
+[2024-08-31 17:14:05,478][Main][INFO] - [train] Step 15575 out of 20000 | Loss --> 1.778 | Grad_l2 --> 0.200 | Weights_l2 --> 11272.340 | Lr --> 0.002 | Seconds_per_step --> 4.878 | 
+[2024-08-31 17:16:07,219][Main][INFO] - [train] Step 15600 out of 20000 | Loss --> 1.780 | Grad_l2 --> 0.196 | Weights_l2 --> 11272.323 | Lr --> 0.002 | Seconds_per_step --> 4.870 | 
+[2024-08-31 17:18:10,558][Main][INFO] - [train] Step 15625 out of 20000 | Loss --> 1.772 | Grad_l2 --> 0.200 | Weights_l2 --> 11272.311 | Lr --> 0.002 | Seconds_per_step --> 4.933 | 
+[2024-08-31 17:20:12,172][Main][INFO] - [train] Step 15650 out of 20000 | Loss --> 1.780 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.303 | Lr --> 0.002 | Seconds_per_step --> 4.864 | 
+[2024-08-31 17:22:13,878][Main][INFO] - [train] Step 15675 out of 20000 | Loss --> 1.767 | Grad_l2 --> 0.196 | Weights_l2 --> 11272.290 | Lr --> 0.002 | Seconds_per_step --> 4.868 | 
+[2024-08-31 17:24:17,279][Main][INFO] - [train] Step 15700 out of 20000 | Loss --> 1.773 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.273 | Lr --> 0.002 | Seconds_per_step --> 4.936 | 
+[2024-08-31 17:26:19,126][Main][INFO] - [train] Step 15725 out of 20000 | Loss --> 1.780 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.263 | Lr --> 0.002 | Seconds_per_step --> 4.874 | 
+[2024-08-31 17:28:20,908][Main][INFO] - [train] Step 15750 out of 20000 | Loss --> 1.782 | Grad_l2 --> 0.200 | Weights_l2 --> 11272.246 | Lr --> 0.002 | Seconds_per_step --> 4.871 | 
+[2024-08-31 17:30:24,238][Main][INFO] - [train] Step 15775 out of 20000 | Loss --> 1.768 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.233 | Lr --> 0.002 | Seconds_per_step --> 4.933 | 
+[2024-08-31 17:32:26,169][Main][INFO] - [train] Step 15800 out of 20000 | Loss --> 1.760 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.220 | Lr --> 0.002 | Seconds_per_step --> 4.877 | 
+[2024-08-31 17:34:28,677][Main][INFO] - [train] Step 15825 out of 20000 | Loss --> 1.766 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.207 | Lr --> 0.002 | Seconds_per_step --> 4.900 | 
+[2024-08-31 17:36:32,092][Main][INFO] - [train] Step 15850 out of 20000 | Loss --> 1.781 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.196 | Lr --> 0.002 | Seconds_per_step --> 4.936 | 
+[2024-08-31 17:38:33,746][Main][INFO] - [train] Step 15875 out of 20000 | Loss --> 1.763 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.177 | Lr --> 0.002 | Seconds_per_step --> 4.866 | 
+[2024-08-31 17:40:35,415][Main][INFO] - [train] Step 15900 out of 20000 | Loss --> 1.770 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.165 | Lr --> 0.002 | Seconds_per_step --> 4.867 | 
+[2024-08-31 17:42:36,852][Main][INFO] - [train] Step 15925 out of 20000 | Loss --> 1.766 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.154 | Lr --> 0.002 | Seconds_per_step --> 4.857 | 
+[2024-08-31 17:44:40,125][Main][INFO] - [train] Step 15950 out of 20000 | Loss --> 1.774 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.141 | Lr --> 0.002 | Seconds_per_step --> 4.931 | 
+[2024-08-31 17:46:42,051][Main][INFO] - [train] Step 15975 out of 20000 | Loss --> 1.761 | Grad_l2 --> 0.196 | Weights_l2 --> 11272.121 | Lr --> 0.002 | Seconds_per_step --> 4.877 | 
+[2024-08-31 17:48:43,831][Main][INFO] - [train] Step 16000 out of 20000 | Loss --> 1.775 | Grad_l2 --> 0.196 | Weights_l2 --> 11272.110 | Lr --> 0.002 | Seconds_per_step --> 4.871 | 
+[2024-08-31 17:50:47,287][Main][INFO] - [train] Step 16025 out of 20000 | Loss --> 1.768 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.098 | Lr --> 0.002 | Seconds_per_step --> 4.938 | 
+[2024-08-31 17:52:48,992][Main][INFO] - [train] Step 16050 out of 20000 | Loss --> 1.770 | Grad_l2 --> 0.202 | Weights_l2 --> 11272.084 | Lr --> 0.002 | Seconds_per_step --> 4.868 | 
+[2024-08-31 17:54:50,724][Main][INFO] - [train] Step 16075 out of 20000 | Loss --> 1.767 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.068 | Lr --> 0.002 | Seconds_per_step --> 4.869 | 
+[2024-08-31 17:56:54,016][Main][INFO] - [train] Step 16100 out of 20000 | Loss --> 1.777 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.054 | Lr --> 0.002 | Seconds_per_step --> 4.932 | 
+[2024-08-31 17:58:55,942][Main][INFO] - [train] Step 16125 out of 20000 | Loss --> 1.768 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.037 | Lr --> 0.002 | Seconds_per_step --> 4.877 | 
+[2024-08-31 18:00:57,478][Main][INFO] - [train] Step 16150 out of 20000 | Loss --> 1.766 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.020 | Lr --> 0.002 | Seconds_per_step --> 4.861 | 
+[2024-08-31 18:03:00,736][Main][INFO] - [train] Step 16175 out of 20000 | Loss --> 1.767 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.000 | Lr --> 0.002 | Seconds_per_step --> 4.930 | 
+[2024-08-31 18:05:02,586][Main][INFO] - [train] Step 16200 out of 20000 | Loss --> 1.760 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.988 | Lr --> 0.002 | Seconds_per_step --> 4.874 | 
+[2024-08-31 18:07:04,438][Main][INFO] - [train] Step 16225 out of 20000 | Loss --> 1.758 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.975 | Lr --> 0.002 | Seconds_per_step --> 4.874 | 
+[2024-08-31 18:09:07,901][Main][INFO] - [train] Step 16250 out of 20000 | Loss --> 1.763 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.962 | Lr --> 0.001 | Seconds_per_step --> 4.938 | 
+[2024-08-31 18:11:09,649][Main][INFO] - [train] Step 16275 out of 20000 | Loss --> 1.749 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.952 | Lr --> 0.001 | Seconds_per_step --> 4.870 | 
+[2024-08-31 18:13:11,767][Main][INFO] - [train] Step 16300 out of 20000 | Loss --> 1.754 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.935 | Lr --> 0.001 | Seconds_per_step --> 4.885 | 
+[2024-08-31 18:15:13,819][Main][INFO] - [train] Step 16325 out of 20000 | Loss --> 1.765 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.928 | Lr --> 0.001 | Seconds_per_step --> 4.882 | 
+[2024-08-31 18:17:17,096][Main][INFO] - [train] Step 16350 out of 20000 | Loss --> 1.749 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.916 | Lr --> 0.001 | Seconds_per_step --> 4.931 | 
+[2024-08-31 18:19:18,879][Main][INFO] - [train] Step 16375 out of 20000 | Loss --> 1.757 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.902 | Lr --> 0.001 | Seconds_per_step --> 4.871 | 
+[2024-08-31 18:21:20,566][Main][INFO] - [train] Step 16400 out of 20000 | Loss --> 1.770 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.886 | Lr --> 0.001 | Seconds_per_step --> 4.867 | 
+[2024-08-31 18:23:24,550][Main][INFO] - [train] Step 16425 out of 20000 | Loss --> 1.752 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.874 | Lr --> 0.001 | Seconds_per_step --> 4.959 | 
+[2024-08-31 18:25:26,461][Main][INFO] - [train] Step 16450 out of 20000 | Loss --> 1.756 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.860 | Lr --> 0.001 | Seconds_per_step --> 4.876 | 
+[2024-08-31 18:27:28,278][Main][INFO] - [train] Step 16475 out of 20000 | Loss --> 1.754 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.846 | Lr --> 0.001 | Seconds_per_step --> 4.873 | 
+[2024-08-31 18:29:32,921][Main][INFO] - [train] Step 16500 out of 20000 | Loss --> 1.753 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.834 | Lr --> 0.001 | Seconds_per_step --> 4.986 | 
+[2024-08-31 18:31:35,149][Main][INFO] - [train] Step 16525 out of 20000 | Loss --> 1.760 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.819 | Lr --> 0.001 | Seconds_per_step --> 4.889 | 
+[2024-08-31 18:33:37,364][Main][INFO] - [train] Step 16550 out of 20000 | Loss --> 1.754 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.799 | Lr --> 0.001 | Seconds_per_step --> 4.889 | 
+[2024-08-31 18:35:40,915][Main][INFO] - [train] Step 16575 out of 20000 | Loss --> 1.751 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.786 | Lr --> 0.001 | Seconds_per_step --> 4.942 | 
+[2024-08-31 18:37:43,466][Main][INFO] - [train] Step 16600 out of 20000 | Loss --> 1.768 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.770 | Lr --> 0.001 | Seconds_per_step --> 4.902 | 
+[2024-08-31 18:39:45,637][Main][INFO] - [train] Step 16625 out of 20000 | Loss --> 1.749 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.754 | Lr --> 0.001 | Seconds_per_step --> 4.887 | 
+[2024-08-31 18:41:49,405][Main][INFO] - [train] Step 16650 out of 20000 | Loss --> 1.748 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.739 | Lr --> 0.001 | Seconds_per_step --> 4.951 | 
+[2024-08-31 18:43:51,102][Main][INFO] - [train] Step 16675 out of 20000 | Loss --> 1.745 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.723 | Lr --> 0.001 | Seconds_per_step --> 4.868 | 
+[2024-08-31 18:45:52,944][Main][INFO] - [train] Step 16700 out of 20000 | Loss --> 1.736 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.708 | Lr --> 0.001 | Seconds_per_step --> 4.874 | 
+[2024-08-31 18:47:56,099][Main][INFO] - [train] Step 16725 out of 20000 | Loss --> 1.757 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.698 | Lr --> 0.001 | Seconds_per_step --> 4.926 | 
+[2024-08-31 18:49:57,801][Main][INFO] - [train] Step 16750 out of 20000 | Loss --> 1.742 | Grad_l2 --> 0.195 | Weights_l2 --> 11271.684 | Lr --> 0.001 | Seconds_per_step --> 4.868 | 
+[2024-08-31 18:51:59,437][Main][INFO] - [train] Step 16775 out of 20000 | Loss --> 1.755 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.665 | Lr --> 0.001 | Seconds_per_step --> 4.865 | 
+[2024-08-31 18:54:00,855][Main][INFO] - [train] Step 16800 out of 20000 | Loss --> 1.747 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.651 | Lr --> 0.001 | Seconds_per_step --> 4.857 | 
+[2024-08-31 18:56:03,853][Main][INFO] - [train] Step 16825 out of 20000 | Loss --> 1.743 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.632 | Lr --> 0.001 | Seconds_per_step --> 4.920 | 
+[2024-08-31 18:58:05,201][Main][INFO] - [train] Step 16850 out of 20000 | Loss --> 1.745 | Grad_l2 --> 0.201 | Weights_l2 --> 11271.619 | Lr --> 0.001 | Seconds_per_step --> 4.854 | 
+[2024-08-31 19:00:06,563][Main][INFO] - [train] Step 16875 out of 20000 | Loss --> 1.753 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.606 | Lr --> 0.001 | Seconds_per_step --> 4.854 | 
+[2024-08-31 19:02:09,317][Main][INFO] - [train] Step 16900 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.591 | Lr --> 0.001 | Seconds_per_step --> 4.910 | 
+[2024-08-31 19:04:10,472][Main][INFO] - [train] Step 16925 out of 20000 | Loss --> 1.729 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.575 | Lr --> 0.001 | Seconds_per_step --> 4.846 | 
+[2024-08-31 19:06:11,583][Main][INFO] - [train] Step 16950 out of 20000 | Loss --> 1.741 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.559 | Lr --> 0.001 | Seconds_per_step --> 4.844 | 
+[2024-08-31 19:08:14,363][Main][INFO] - [train] Step 16975 out of 20000 | Loss --> 1.729 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.545 | Lr --> 0.001 | Seconds_per_step --> 4.911 | 
+[2024-08-31 19:10:15,178][Main][INFO] - [train] Step 17000 out of 20000 | Loss --> 1.738 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.533 | Lr --> 0.001 | Seconds_per_step --> 4.832 | 
+[2024-08-31 19:12:16,110][Main][INFO] - [train] Step 17025 out of 20000 | Loss --> 1.741 | Grad_l2 --> 0.200 | Weights_l2 --> 11271.515 | Lr --> 0.001 | Seconds_per_step --> 4.837 | 
+[2024-08-31 19:14:18,572][Main][INFO] - [train] Step 17050 out of 20000 | Loss --> 1.740 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.500 | Lr --> 0.001 | Seconds_per_step --> 4.898 | 
+[2024-08-31 19:16:19,446][Main][INFO] - [train] Step 17075 out of 20000 | Loss --> 1.735 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.485 | Lr --> 0.001 | Seconds_per_step --> 4.835 | 
+[2024-08-31 19:18:20,449][Main][INFO] - [train] Step 17100 out of 20000 | Loss --> 1.734 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.471 | Lr --> 0.001 | Seconds_per_step --> 4.840 | 
+[2024-08-31 19:20:23,025][Main][INFO] - [train] Step 17125 out of 20000 | Loss --> 1.736 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.453 | Lr --> 0.001 | Seconds_per_step --> 4.903 | 
+[2024-08-31 19:22:23,946][Main][INFO] - [train] Step 17150 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.439 | Lr --> 0.001 | Seconds_per_step --> 4.837 | 
+[2024-08-31 19:24:25,361][Main][INFO] - [train] Step 17175 out of 20000 | Loss --> 1.732 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.422 | Lr --> 0.001 | Seconds_per_step --> 4.857 | 
+[2024-08-31 19:26:26,446][Main][INFO] - [train] Step 17200 out of 20000 | Loss --> 1.734 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.408 | Lr --> 0.001 | Seconds_per_step --> 4.843 | 
+[2024-08-31 19:28:29,313][Main][INFO] - [train] Step 17225 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.395 | Lr --> 0.001 | Seconds_per_step --> 4.915 | 
+[2024-08-31 19:30:30,859][Main][INFO] - [train] Step 17250 out of 20000 | Loss --> 1.737 | Grad_l2 --> 0.195 | Weights_l2 --> 11271.382 | Lr --> 0.001 | Seconds_per_step --> 4.862 | 
+[2024-08-31 19:32:32,558][Main][INFO] - [train] Step 17275 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.367 | Lr --> 0.001 | Seconds_per_step --> 4.868 | 
+[2024-08-31 19:34:35,555][Main][INFO] - [train] Step 17300 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.349 | Lr --> 0.001 | Seconds_per_step --> 4.920 | 
+[2024-08-31 19:36:37,051][Main][INFO] - [train] Step 17325 out of 20000 | Loss --> 1.722 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.333 | Lr --> 0.001 | Seconds_per_step --> 4.860 | 
+[2024-08-31 19:38:38,163][Main][INFO] - [train] Step 17350 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.315 | Lr --> 0.001 | Seconds_per_step --> 4.844 | 
+[2024-08-31 19:40:41,121][Main][INFO] - [train] Step 17375 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.299 | Lr --> 0.001 | Seconds_per_step --> 4.918 | 
+[2024-08-31 19:42:42,654][Main][INFO] - [train] Step 17400 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.201 | Weights_l2 --> 11271.281 | Lr --> 0.001 | Seconds_per_step --> 4.861 | 
+[2024-08-31 19:44:44,126][Main][INFO] - [train] Step 17425 out of 20000 | Loss --> 1.713 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.264 | Lr --> 0.001 | Seconds_per_step --> 4.859 | 
+[2024-08-31 19:46:47,266][Main][INFO] - [train] Step 17450 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.200 | Weights_l2 --> 11271.245 | Lr --> 0.001 | Seconds_per_step --> 4.925 | 
+[2024-08-31 19:48:48,692][Main][INFO] - [train] Step 17475 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.227 | Lr --> 0.001 | Seconds_per_step --> 4.857 | 
+[2024-08-31 19:50:50,523][Main][INFO] - [train] Step 17500 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.212 | Lr --> 0.001 | Seconds_per_step --> 4.873 | 
+[2024-08-31 19:50:50,523][accelerate.accelerator][INFO] - Saving current state to checkpoint-pt-17500
+[2024-08-31 19:50:50,530][accelerate.utils.other][WARNING] - Removed shared tensor {'decoder.embed_tokens.weight', 'encoder.embed_tokens.weight'} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading
+[2024-08-31 19:50:57,172][accelerate.checkpointing][INFO] - Model weights saved in checkpoint-pt-17500/model.safetensors
+[2024-08-31 19:51:06,254][accelerate.checkpointing][INFO] - Optimizer state saved in checkpoint-pt-17500/optimizer.bin
+[2024-08-31 19:51:06,257][accelerate.checkpointing][INFO] - Scheduler state saved in checkpoint-pt-17500/scheduler.bin
+[2024-08-31 19:51:06,258][accelerate.checkpointing][INFO] - Sampler state for dataloader 0 saved in checkpoint-pt-17500/sampler.bin
+[2024-08-31 19:51:06,260][accelerate.checkpointing][INFO] - Sampler state for dataloader 1 saved in checkpoint-pt-17500/sampler_1.bin
+[2024-08-31 19:51:06,261][accelerate.checkpointing][INFO] - Random states saved in checkpoint-pt-17500/random_states_0.pkl
+[2024-08-31 19:53:08,757][Main][INFO] - [train] Step 17525 out of 20000 | Loss --> 1.716 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.193 | Lr --> 0.001 | Seconds_per_step --> 5.529 | 
+[2024-08-31 19:55:09,923][Main][INFO] - [train] Step 17550 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.173 | Lr --> 0.001 | Seconds_per_step --> 4.847 | 
+[2024-08-31 19:57:11,243][Main][INFO] - [train] Step 17575 out of 20000 | Loss --> 1.700 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.155 | Lr --> 0.001 | Seconds_per_step --> 4.853 | 
+[2024-08-31 19:59:14,099][Main][INFO] - [train] Step 17600 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.137 | Lr --> 0.001 | Seconds_per_step --> 4.914 | 
+[2024-08-31 20:01:15,562][Main][INFO] - [train] Step 17625 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.118 | Lr --> 0.001 | Seconds_per_step --> 4.858 | 
+[2024-08-31 20:03:16,470][Main][INFO] - [train] Step 17650 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.097 | Lr --> 0.001 | Seconds_per_step --> 4.836 | 
+[2024-08-31 20:05:17,916][Main][INFO] - [train] Step 17675 out of 20000 | Loss --> 1.733 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.078 | Lr --> 0.001 | Seconds_per_step --> 4.858 | 
+[2024-08-31 20:07:20,683][Main][INFO] - [train] Step 17700 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.062 | Lr --> 0.001 | Seconds_per_step --> 4.911 | 
+[2024-08-31 20:09:22,414][Main][INFO] - [train] Step 17725 out of 20000 | Loss --> 1.707 | Grad_l2 --> 0.195 | Weights_l2 --> 11271.041 | Lr --> 0.001 | Seconds_per_step --> 4.869 | 
+[2024-08-31 20:11:24,033][Main][INFO] - [train] Step 17750 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.024 | Lr --> 0.001 | Seconds_per_step --> 4.865 | 
+[2024-08-31 20:13:26,602][Main][INFO] - [train] Step 17775 out of 20000 | Loss --> 1.730 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.007 | Lr --> 0.001 | Seconds_per_step --> 4.903 | 
+[2024-08-31 20:15:27,607][Main][INFO] - [train] Step 17800 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.991 | Lr --> 0.001 | Seconds_per_step --> 4.840 | 
+[2024-08-31 20:17:28,616][Main][INFO] - [train] Step 17825 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.976 | Lr --> 0.001 | Seconds_per_step --> 4.840 | 
+[2024-08-31 20:19:31,033][Main][INFO] - [train] Step 17850 out of 20000 | Loss --> 1.729 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.961 | Lr --> 0.001 | Seconds_per_step --> 4.897 | 
+[2024-08-31 20:21:32,133][Main][INFO] - [train] Step 17875 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.946 | Lr --> 0.001 | Seconds_per_step --> 4.844 | 
+[2024-08-31 20:23:33,151][Main][INFO] - [train] Step 17900 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.933 | Lr --> 0.000 | Seconds_per_step --> 4.841 | 
+[2024-08-31 20:25:35,828][Main][INFO] - [train] Step 17925 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.921 | Lr --> 0.000 | Seconds_per_step --> 4.907 | 
+[2024-08-31 20:27:36,892][Main][INFO] - [train] Step 17950 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.911 | Lr --> 0.000 | Seconds_per_step --> 4.842 | 
+[2024-08-31 20:29:38,066][Main][INFO] - [train] Step 17975 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.900 | Lr --> 0.000 | Seconds_per_step --> 4.847 | 
+[2024-08-31 20:31:40,569][Main][INFO] - [train] Step 18000 out of 20000 | Loss --> 1.738 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.891 | Lr --> 0.000 | Seconds_per_step --> 4.900 | 
+[2024-08-31 20:33:41,408][Main][INFO] - [train] Step 18025 out of 20000 | Loss --> 1.739 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.884 | Lr --> 0.000 | Seconds_per_step --> 4.833 | 
+[2024-08-31 20:35:42,352][Main][INFO] - [train] Step 18050 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.876 | Lr --> 0.000 | Seconds_per_step --> 4.838 | 
+[2024-08-31 20:37:45,322][Main][INFO] - [train] Step 18075 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.869 | Lr --> 0.000 | Seconds_per_step --> 4.919 | 
+[2024-08-31 20:39:46,981][Main][INFO] - [train] Step 18100 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.862 | Lr --> 0.000 | Seconds_per_step --> 4.866 | 
+[2024-08-31 20:41:48,584][Main][INFO] - [train] Step 18125 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.855 | Lr --> 0.000 | Seconds_per_step --> 4.864 | 
+[2024-08-31 20:43:49,907][Main][INFO] - [train] Step 18150 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.853 | 
+[2024-08-31 20:45:52,968][Main][INFO] - [train] Step 18175 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.844 | Lr --> 0.000 | Seconds_per_step --> 4.922 | 
+[2024-08-31 20:47:54,325][Main][INFO] - [train] Step 18200 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.854 | 
+[2024-08-31 20:49:55,663][Main][INFO] - [train] Step 18225 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.195 | Weights_l2 --> 11270.840 | Lr --> 0.000 | Seconds_per_step --> 4.853 | 
+[2024-08-31 20:51:58,657][Main][INFO] - [train] Step 18250 out of 20000 | Loss --> 1.718 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.836 | Lr --> 0.000 | Seconds_per_step --> 4.920 | 
+[2024-08-31 20:54:00,083][Main][INFO] - [train] Step 18275 out of 20000 | Loss --> 1.721 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.834 | Lr --> 0.000 | Seconds_per_step --> 4.857 | 
+[2024-08-31 20:56:01,850][Main][INFO] - [train] Step 18300 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.831 | Lr --> 0.000 | Seconds_per_step --> 4.871 | 
+[2024-08-31 20:58:04,690][Main][INFO] - [train] Step 18325 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.832 | Lr --> 0.000 | Seconds_per_step --> 4.913 | 
+[2024-08-31 21:00:06,226][Main][INFO] - [train] Step 18350 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.832 | Lr --> 0.000 | Seconds_per_step --> 4.861 | 
+[2024-08-31 21:02:07,970][Main][INFO] - [train] Step 18375 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.831 | Lr --> 0.000 | Seconds_per_step --> 4.870 | 
+[2024-08-31 21:04:11,035][Main][INFO] - [train] Step 18400 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.831 | Lr --> 0.000 | Seconds_per_step --> 4.922 | 
+[2024-08-31 21:06:12,731][Main][INFO] - [train] Step 18425 out of 20000 | Loss --> 1.714 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.834 | Lr --> 0.000 | Seconds_per_step --> 4.868 | 
+[2024-08-31 21:08:14,292][Main][INFO] - [train] Step 18450 out of 20000 | Loss --> 1.721 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.836 | Lr --> 0.000 | Seconds_per_step --> 4.862 | 
+[2024-08-31 21:10:17,481][Main][INFO] - [train] Step 18475 out of 20000 | Loss --> 1.714 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.837 | Lr --> 0.000 | Seconds_per_step --> 4.927 | 
+[2024-08-31 21:12:19,115][Main][INFO] - [train] Step 18500 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.838 | Lr --> 0.000 | Seconds_per_step --> 4.865 | 
+[2024-08-31 21:14:20,604][Main][INFO] - [train] Step 18525 out of 20000 | Loss --> 1.714 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.839 | Lr --> 0.000 | Seconds_per_step --> 4.859 | 
+[2024-08-31 21:16:24,832][Main][INFO] - [train] Step 18550 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.969 | 
+[2024-08-31 21:18:26,217][Main][INFO] - [train] Step 18575 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.855 | 
+[2024-08-31 21:20:27,684][Main][INFO] - [train] Step 18600 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.840 | Lr --> 0.000 | Seconds_per_step --> 4.859 | 
+[2024-08-31 21:22:29,530][Main][INFO] - [train] Step 18625 out of 20000 | Loss --> 1.730 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.874 | 
+[2024-08-31 21:24:33,115][Main][INFO] - [train] Step 18650 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.195 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.943 | 
+[2024-08-31 21:26:35,121][Main][INFO] - [train] Step 18675 out of 20000 | Loss --> 1.732 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.842 | Lr --> 0.000 | Seconds_per_step --> 4.880 | 
+[2024-08-31 21:28:37,128][Main][INFO] - [train] Step 18700 out of 20000 | Loss --> 1.722 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.843 | Lr --> 0.000 | Seconds_per_step --> 4.880 | 
+[2024-08-31 21:30:40,396][Main][INFO] - [train] Step 18725 out of 20000 | Loss --> 1.712 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.843 | Lr --> 0.000 | Seconds_per_step --> 4.931 | 
+[2024-08-31 21:33:03,172][Main][INFO] - [train] Step 18750 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.844 | Lr --> 0.000 | Seconds_per_step --> 5.711 | 
+[2024-08-31 21:35:04,721][Main][INFO] - [train] Step 18775 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.845 | Lr --> 0.000 | Seconds_per_step --> 4.862 | 
+[2024-08-31 21:37:08,128][Main][INFO] - [train] Step 18800 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.845 | Lr --> 0.000 | Seconds_per_step --> 4.936 | 
+[2024-08-31 21:39:09,857][Main][INFO] - [train] Step 18825 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.201 | Weights_l2 --> 11270.845 | Lr --> 0.000 | Seconds_per_step --> 4.869 | 
+[2024-08-31 21:41:11,700][Main][INFO] - [train] Step 18850 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.847 | Lr --> 0.000 | Seconds_per_step --> 4.874 | 
+[2024-08-31 21:43:15,117][Main][INFO] - [train] Step 18875 out of 20000 | Loss --> 1.722 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.846 | Lr --> 0.000 | Seconds_per_step --> 4.937 | 
+[2024-08-31 21:45:19,433][Main][INFO] - [train] Step 18900 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.847 | Lr --> 0.000 | Seconds_per_step --> 4.973 | 
+[2024-08-31 21:47:29,032][Main][INFO] - [train] Step 18925 out of 20000 | Loss --> 1.709 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.847 | Lr --> 0.000 | Seconds_per_step --> 5.184 | 
+[2024-08-31 21:49:33,512][Main][INFO] - [train] Step 18950 out of 20000 | Loss --> 1.731 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.848 | Lr --> 0.000 | Seconds_per_step --> 4.979 | 
+[2024-08-31 21:51:35,196][Main][INFO] - [train] Step 18975 out of 20000 | Loss --> 1.721 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.867 | 
+[2024-08-31 21:53:36,788][Main][INFO] - [train] Step 19000 out of 20000 | Loss --> 1.718 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.864 | 
+[2024-08-31 21:55:38,313][Main][INFO] - [train] Step 19025 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.861 | 
+[2024-08-31 21:57:41,329][Main][INFO] - [train] Step 19050 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.921 | 
+[2024-08-31 21:59:42,853][Main][INFO] - [train] Step 19075 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.861 | 
+[2024-08-31 22:01:44,492][Main][INFO] - [train] Step 19100 out of 20000 | Loss --> 1.730 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.865 | 
+[2024-08-31 22:03:47,660][Main][INFO] - [train] Step 19125 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.927 | 
+[2024-08-31 22:05:49,133][Main][INFO] - [train] Step 19150 out of 20000 | Loss --> 1.712 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.859 | 
+[2024-08-31 22:07:50,623][Main][INFO] - [train] Step 19175 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.860 | 
+[2024-08-31 22:09:53,873][Main][INFO] - [train] Step 19200 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.930 | 
+[2024-08-31 22:11:55,529][Main][INFO] - [train] Step 19225 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.866 | 
+[2024-08-31 22:13:57,272][Main][INFO] - [train] Step 19250 out of 20000 | Loss --> 1.730 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.870 | 
+[2024-08-31 22:16:01,229][Main][INFO] - [train] Step 19275 out of 20000 | Loss --> 1.722 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.958 | 
+[2024-08-31 22:18:03,766][Main][INFO] - [train] Step 19300 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.901 | 
+[2024-08-31 22:20:06,053][Main][INFO] - [train] Step 19325 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.891 | 
+[2024-08-31 22:22:09,832][Main][INFO] - [train] Step 19350 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.200 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.951 | 
+[2024-08-31 22:24:11,539][Main][INFO] - [train] Step 19375 out of 20000 | Loss --> 1.707 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.868 | 
+[2024-08-31 22:26:13,254][Main][INFO] - [train] Step 19400 out of 20000 | Loss --> 1.718 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.869 | 
+[2024-08-31 22:28:15,089][Main][INFO] - [train] Step 19425 out of 20000 | Loss --> 1.711 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.873 | 
+[2024-08-31 22:30:18,617][Main][INFO] - [train] Step 19450 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.941 | 
+[2024-08-31 22:32:20,460][Main][INFO] - [train] Step 19475 out of 20000 | Loss --> 1.711 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.874 | 
+[2024-08-31 22:34:21,991][Main][INFO] - [train] Step 19500 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.861 | 
+[2024-08-31 22:36:24,959][Main][INFO] - [train] Step 19525 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.919 | 
+[2024-08-31 22:38:26,479][Main][INFO] - [train] Step 19550 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.861 | 
+[2024-08-31 22:40:28,432][Main][INFO] - [train] Step 19575 out of 20000 | Loss --> 1.714 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.878 | 
+[2024-08-31 22:42:31,962][Main][INFO] - [train] Step 19600 out of 20000 | Loss --> 1.718 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.941 | 
+[2024-08-31 22:44:33,970][Main][INFO] - [train] Step 19625 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.880 | 
+[2024-08-31 22:46:38,990][Main][INFO] - [train] Step 19650 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 5.001 | 
+[2024-08-31 22:48:42,541][Main][INFO] - [train] Step 19675 out of 20000 | Loss --> 1.711 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.942 | 
+[2024-08-31 22:50:48,499][Main][INFO] - [train] Step 19700 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 5.038 | 
+[2024-08-31 22:52:50,152][Main][INFO] - [train] Step 19725 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.866 | 
+[2024-08-31 22:54:53,252][Main][INFO] - [train] Step 19750 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.924 | 
+[2024-08-31 22:56:55,118][Main][INFO] - [train] Step 19775 out of 20000 | Loss --> 1.712 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.875 | 
+[2024-08-31 22:58:56,863][Main][INFO] - [train] Step 19800 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.870 | 
+[2024-08-31 23:01:01,950][Main][INFO] - [train] Step 19825 out of 20000 | Loss --> 1.710 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 5.003 | 
+[2024-08-31 23:03:04,913][Main][INFO] - [train] Step 19850 out of 20000 | Loss --> 1.713 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.918 | 
+[2024-08-31 23:05:06,946][Main][INFO] - [train] Step 19875 out of 20000 | Loss --> 1.710 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.881 | 
+[2024-08-31 23:07:08,902][Main][INFO] - [train] Step 19900 out of 20000 | Loss --> 1.711 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.878 | 
+[2024-08-31 23:09:12,065][Main][INFO] - [train] Step 19925 out of 20000 | Loss --> 1.716 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.926 | 
+[2024-08-31 23:11:13,586][Main][INFO] - [train] Step 19950 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.861 | 
+[2024-08-31 23:13:15,435][Main][INFO] - [train] Step 19975 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.874 | 
+[2024-08-31 23:15:18,590][Main][INFO] - [train] Step 20000 out of 20000 | Loss --> 1.716 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.926 | 
+[2024-08-31 23:15:18,591][accelerate.accelerator][INFO] - Saving current state to checkpoint-pt-20000
+[2024-08-31 23:15:18,599][accelerate.utils.other][WARNING] - Removed shared tensor {'decoder.embed_tokens.weight', 'encoder.embed_tokens.weight'} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading
+[2024-08-31 23:15:26,324][accelerate.checkpointing][INFO] - Model weights saved in checkpoint-pt-20000/model.safetensors
+[2024-08-31 23:15:35,439][accelerate.checkpointing][INFO] - Optimizer state saved in checkpoint-pt-20000/optimizer.bin
+[2024-08-31 23:15:35,440][accelerate.checkpointing][INFO] - Scheduler state saved in checkpoint-pt-20000/scheduler.bin
+[2024-08-31 23:15:35,441][accelerate.checkpointing][INFO] - Sampler state for dataloader 0 saved in checkpoint-pt-20000/sampler.bin
+[2024-08-31 23:15:35,441][accelerate.checkpointing][INFO] - Sampler state for dataloader 1 saved in checkpoint-pt-20000/sampler_1.bin
+[2024-08-31 23:15:35,442][accelerate.checkpointing][INFO] - Random states saved in checkpoint-pt-20000/random_states_0.pkl
+[2024-08-31 23:15:39,524][datasets.iterable_dataset][WARNING] - Too many dataloader workers: 16 (max is dataset.n_shards=8). Stopping 8 dataloader workers.
+[2024-08-31 23:31:42,282][Main][INFO] - [eval] Step 20001 out of 20000 | Loss --> 2.073 | Accuracy --> 0.604 | Time --> 964.275 | 
+[2024-08-31 23:31:42,287][accelerate.accelerator][INFO] - Saving current state to checkpoint-pt-20001
+[2024-08-31 23:31:42,295][accelerate.utils.other][WARNING] - Removed shared tensor {'decoder.embed_tokens.weight', 'encoder.embed_tokens.weight'} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading
+[2024-08-31 23:31:50,975][accelerate.checkpointing][INFO] - Model weights saved in checkpoint-pt-20001/model.safetensors
+[2024-08-31 23:32:00,717][accelerate.checkpointing][INFO] - Optimizer state saved in checkpoint-pt-20001/optimizer.bin
+[2024-08-31 23:32:00,719][accelerate.checkpointing][INFO] - Scheduler state saved in checkpoint-pt-20001/scheduler.bin
+[2024-08-31 23:32:00,720][accelerate.checkpointing][INFO] - Sampler state for dataloader 0 saved in checkpoint-pt-20001/sampler.bin
+[2024-08-31 23:32:00,720][accelerate.checkpointing][INFO] - Sampler state for dataloader 1 saved in checkpoint-pt-20001/sampler_1.bin
+[2024-08-31 23:32:00,721][accelerate.checkpointing][INFO] - Random states saved in checkpoint-pt-20001/random_states_0.pkl

checkpoints/tokenizer/added_tokens.json

@@ -0,0 +1,102 @@
+{
+  "<extra_id_0>": 48128,
+  "<extra_id_10>": 48138,
+  "<extra_id_11>": 48139,
+  "<extra_id_12>": 48140,
+  "<extra_id_13>": 48141,
+  "<extra_id_14>": 48142,
+  "<extra_id_15>": 48143,
+  "<extra_id_16>": 48144,
+  "<extra_id_17>": 48145,
+  "<extra_id_18>": 48146,
+  "<extra_id_19>": 48147,
+  "<extra_id_1>": 48129,
+  "<extra_id_20>": 48148,
+  "<extra_id_21>": 48149,
+  "<extra_id_22>": 48150,
+  "<extra_id_23>": 48151,
+  "<extra_id_24>": 48152,
+  "<extra_id_25>": 48153,
+  "<extra_id_26>": 48154,
+  "<extra_id_27>": 48155,
+  "<extra_id_28>": 48156,
+  "<extra_id_29>": 48157,
+  "<extra_id_2>": 48130,
+  "<extra_id_30>": 48158,
+  "<extra_id_31>": 48159,
+  "<extra_id_32>": 48160,
+  "<extra_id_33>": 48161,
+  "<extra_id_34>": 48162,
+  "<extra_id_35>": 48163,
+  "<extra_id_36>": 48164,
+  "<extra_id_37>": 48165,
+  "<extra_id_38>": 48166,
+  "<extra_id_39>": 48167,
+  "<extra_id_3>": 48131,
+  "<extra_id_40>": 48168,
+  "<extra_id_41>": 48169,
+  "<extra_id_42>": 48170,
+  "<extra_id_43>": 48171,
+  "<extra_id_44>": 48172,
+  "<extra_id_45>": 48173,
+  "<extra_id_46>": 48174,
+  "<extra_id_47>": 48175,
+  "<extra_id_48>": 48176,
+  "<extra_id_49>": 48177,
+  "<extra_id_4>": 48132,
+  "<extra_id_50>": 48178,
+  "<extra_id_51>": 48179,
+  "<extra_id_52>": 48180,
+  "<extra_id_53>": 48181,
+  "<extra_id_54>": 48182,
+  "<extra_id_55>": 48183,
+  "<extra_id_56>": 48184,
+  "<extra_id_57>": 48185,
+  "<extra_id_58>": 48186,
+  "<extra_id_59>": 48187,
+  "<extra_id_5>": 48133,
+  "<extra_id_60>": 48188,
+  "<extra_id_61>": 48189,
+  "<extra_id_62>": 48190,
+  "<extra_id_63>": 48191,
+  "<extra_id_64>": 48192,
+  "<extra_id_65>": 48193,
+  "<extra_id_66>": 48194,
+  "<extra_id_67>": 48195,
+  "<extra_id_68>": 48196,
+  "<extra_id_69>": 48197,
+  "<extra_id_6>": 48134,
+  "<extra_id_70>": 48198,
+  "<extra_id_71>": 48199,
+  "<extra_id_72>": 48200,
+  "<extra_id_73>": 48201,
+  "<extra_id_74>": 48202,
+  "<extra_id_75>": 48203,
+  "<extra_id_76>": 48204,
+  "<extra_id_77>": 48205,
+  "<extra_id_78>": 48206,
+  "<extra_id_79>": 48207,
+  "<extra_id_7>": 48135,
+  "<extra_id_80>": 48208,
+  "<extra_id_81>": 48209,
+  "<extra_id_82>": 48210,
+  "<extra_id_83>": 48211,
+  "<extra_id_84>": 48212,
+  "<extra_id_85>": 48213,
+  "<extra_id_86>": 48214,
+  "<extra_id_87>": 48215,
+  "<extra_id_88>": 48216,
+  "<extra_id_89>": 48217,
+  "<extra_id_8>": 48136,
+  "<extra_id_90>": 48218,
+  "<extra_id_91>": 48219,
+  "<extra_id_92>": 48220,
+  "<extra_id_93>": 48221,
+  "<extra_id_94>": 48222,
+  "<extra_id_95>": 48223,
+  "<extra_id_96>": 48224,
+  "<extra_id_97>": 48225,
+  "<extra_id_98>": 48226,
+  "<extra_id_99>": 48227,
+  "<extra_id_9>": 48137
+}

checkpoints/tokenizer/special_tokens_map.json

@@ -0,0 +1,132 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/tokenizer/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:165515d67d17852d3e7c30ec22d7a745330f2cba264f40c3ea9380eddf84396f
+size 1042483

checkpoints/tokenizer/tokenizer_config.json

@@ -0,0 +1,952 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": true,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48128": {
+      "content": "<extra_id_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48129": {
+      "content": "<extra_id_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48130": {
+      "content": "<extra_id_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48131": {
+      "content": "<extra_id_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48132": {
+      "content": "<extra_id_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48133": {
+      "content": "<extra_id_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48134": {
+      "content": "<extra_id_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48135": {
+      "content": "<extra_id_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48136": {
+      "content": "<extra_id_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48137": {
+      "content": "<extra_id_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48138": {
+      "content": "<extra_id_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48139": {
+      "content": "<extra_id_11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48140": {
+      "content": "<extra_id_12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48141": {
+      "content": "<extra_id_13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48142": {
+      "content": "<extra_id_14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48143": {
+      "content": "<extra_id_15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48144": {
+      "content": "<extra_id_16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48145": {
+      "content": "<extra_id_17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48146": {
+      "content": "<extra_id_18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48147": {
+      "content": "<extra_id_19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48148": {
+      "content": "<extra_id_20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48149": {
+      "content": "<extra_id_21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48150": {
+      "content": "<extra_id_22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48151": {
+      "content": "<extra_id_23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48152": {
+      "content": "<extra_id_24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48153": {
+      "content": "<extra_id_25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48154": {
+      "content": "<extra_id_26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48155": {
+      "content": "<extra_id_27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48156": {
+      "content": "<extra_id_28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48157": {
+      "content": "<extra_id_29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48158": {
+      "content": "<extra_id_30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48159": {
+      "content": "<extra_id_31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48160": {
+      "content": "<extra_id_32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48161": {
+      "content": "<extra_id_33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48162": {
+      "content": "<extra_id_34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48163": {
+      "content": "<extra_id_35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48164": {
+      "content": "<extra_id_36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48165": {
+      "content": "<extra_id_37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48166": {
+      "content": "<extra_id_38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48167": {
+      "content": "<extra_id_39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48168": {
+      "content": "<extra_id_40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48169": {
+      "content": "<extra_id_41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48170": {
+      "content": "<extra_id_42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48171": {
+      "content": "<extra_id_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48172": {
+      "content": "<extra_id_44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48173": {
+      "content": "<extra_id_45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48174": {
+      "content": "<extra_id_46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48175": {
+      "content": "<extra_id_47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48176": {
+      "content": "<extra_id_48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48177": {
+      "content": "<extra_id_49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48178": {
+      "content": "<extra_id_50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48179": {
+      "content": "<extra_id_51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48180": {
+      "content": "<extra_id_52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48181": {
+      "content": "<extra_id_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48182": {
+      "content": "<extra_id_54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48183": {
+      "content": "<extra_id_55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48184": {
+      "content": "<extra_id_56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48185": {
+      "content": "<extra_id_57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48186": {
+      "content": "<extra_id_58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48187": {
+      "content": "<extra_id_59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48188": {
+      "content": "<extra_id_60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48189": {
+      "content": "<extra_id_61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48190": {
+      "content": "<extra_id_62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48191": {
+      "content": "<extra_id_63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48192": {
+      "content": "<extra_id_64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48193": {
+      "content": "<extra_id_65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48194": {
+      "content": "<extra_id_66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48195": {
+      "content": "<extra_id_67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48196": {
+      "content": "<extra_id_68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48197": {
+      "content": "<extra_id_69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48198": {
+      "content": "<extra_id_70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48199": {
+      "content": "<extra_id_71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48200": {
+      "content": "<extra_id_72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48201": {
+      "content": "<extra_id_73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48202": {
+      "content": "<extra_id_74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48203": {
+      "content": "<extra_id_75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48204": {
+      "content": "<extra_id_76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48205": {
+      "content": "<extra_id_77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48206": {
+      "content": "<extra_id_78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48207": {
+      "content": "<extra_id_79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48208": {
+      "content": "<extra_id_80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48209": {
+      "content": "<extra_id_81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48210": {
+      "content": "<extra_id_82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48211": {
+      "content": "<extra_id_83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48212": {
+      "content": "<extra_id_84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48213": {
+      "content": "<extra_id_85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48214": {
+      "content": "<extra_id_86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48215": {
+      "content": "<extra_id_87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48216": {
+      "content": "<extra_id_88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48217": {
+      "content": "<extra_id_89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48218": {
+      "content": "<extra_id_90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48219": {
+      "content": "<extra_id_91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48220": {
+      "content": "<extra_id_92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48221": {
+      "content": "<extra_id_93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48222": {
+      "content": "<extra_id_94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48223": {
+      "content": "<extra_id_95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48224": {
+      "content": "<extra_id_96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48225": {
+      "content": "<extra_id_97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48226": {
+      "content": "<extra_id_98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48227": {
+      "content": "<extra_id_99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoints/wandb/debug.log

@@ -25,3 +25,12 @@ config: {'mode': 'pt', 'device': 'gpu', 'precision': 'bf16', 'eval_only': False,
 2024-08-30 19:59:24,815 INFO    MainThread:29052 [wandb_run.py:_redirect():2399] Redirects installed.
 2024-08-30 19:59:24,818 INFO    MainThread:29052 [wandb_init.py:init():894] run started, returning control to user process
 2024-08-30 19:59:44,796 INFO    MainThread:29052 [wandb_run.py:_config_callback():1392] config_cb None None {'mode': 'pt', 'device': 'gpu', 'precision': 'bf16', 'eval_only': False, 'predict_only': False, 'seed': 34534, 'model': {'klass': 'hf_t5', 'name': 'pszemraj/tFINE-900m-e16-d32', 'overwrite': {'dropout_rate': 0.0}, 'checkpoint_path': '', 'random_init': False, 'compile': True}, 'tokenizer': {'name': 'BEE-spoke-data/slimpajama_tok-48128-BPE-forT5'}, 'data': {'input_length': 1024, 'mlm_probability': 0.15, 'mean_noise_span_length': 3.0, 'num_workers': 16, 'before_mask_input_length': 1137, 'target_length': 229}, 'optim': {'name': 'adamwscale', 'base_lr': 0.01, 'batch_size': 128, 'total_steps': 20000, 'epochs': -1, 'warmup_steps': 5000, 'lr_scheduler': 'cosine', 'weight_decay': 0.0001, 'grad_clip': 1.0, 'grad_acc': 8, 'final_cosine': 2e-05}, 'eval': {'every_steps': 1000000000, 'steps': 500, 'corrected_steps': 500}, 'checkpoint': {'every_steps': 2500}, 'logging': {'use_wandb': True, 'wandb_config': {'project': 'nanoT5', 'entity': 'pszemraj', 'tags': ['900m', '1024'], 'mode': 'online'}, 'every_steps': 25, 'grad_l2': True, 'weights_l2': True}, 'slurm_id': 'none', 'working_dir': '/workspace/nanoT5/outputs/2024-08-30/19-59-22', 'n_all_param': 887492096}
+2024-08-31 23:32:00,793 INFO    MainThread:29052 [wandb_run.py:_finish():2160] finishing run pszemraj/nanoT5/mao0tqjy
+2024-08-31 23:32:00,796 INFO    MainThread:29052 [wandb_run.py:_atexit_cleanup():2424] got exitcode: 0
+2024-08-31 23:32:00,797 INFO    MainThread:29052 [wandb_run.py:_restore():2406] restore
+2024-08-31 23:32:00,798 INFO    MainThread:29052 [wandb_run.py:_restore():2412] restore done
+2024-08-31 23:32:00,799 INFO    MainThread:29052 [wandb_run.py:_on_finish():2677] communicating current version
+2024-08-31 23:32:00,827 INFO    MainThread:29052 [wandb_run.py:_on_finish():2686] got version response 
+2024-08-31 23:32:06,426 INFO    MainThread:29052 [wandb_run.py:_footer_history_summary_info():4078] rendering history
+2024-08-31 23:32:06,427 INFO    MainThread:29052 [wandb_run.py:_footer_history_summary_info():4110] rendering summary
+2024-08-31 23:32:06,433 INFO    MainThread:29052 [wandb_run.py:_footer_sync_info():4037] logging synced files

checkpoints/wandb/run-20240830_195924-mao0tqjy/files/config.yaml

@@ -117,6 +117,7 @@ _wandb:
       - 71
       - 100
       3:
+      - 2
       - 15
       - 16
       - 23

checkpoints/wandb/run-20240830_195924-mao0tqjy/files/output.log

@@ -747,3 +747,209 @@ W0830 20:03:56.180000 136239311107648 torch/fx/experimental/symbolic_shapes.py:4
 [2024-08-31 17:05:56,534][Main][INFO] - [train] Step 15475 out of 20000 | Loss --> 1.777 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.386 | Lr --> 0.002 | Seconds_per_step --> 4.936 |
 [2024-08-31 17:07:58,425][Main][INFO] - [train] Step 15500 out of 20000 | Loss --> 1.773 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.378 | Lr --> 0.002 | Seconds_per_step --> 4.876 |
 [2024-08-31 17:10:00,358][Main][INFO] - [train] Step 15525 out of 20000 | Loss --> 1.778 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.364 | Lr --> 0.002 | Seconds_per_step --> 4.877 |
+[2024-08-31 17:12:03,531][Main][INFO] - [train] Step 15550 out of 20000 | Loss --> 1.773 | Grad_l2 --> 0.200 | Weights_l2 --> 11272.349 | Lr --> 0.002 | Seconds_per_step --> 4.927 |
+[2024-08-31 17:14:05,478][Main][INFO] - [train] Step 15575 out of 20000 | Loss --> 1.778 | Grad_l2 --> 0.200 | Weights_l2 --> 11272.340 | Lr --> 0.002 | Seconds_per_step --> 4.878 |
+[2024-08-31 17:16:07,219][Main][INFO] - [train] Step 15600 out of 20000 | Loss --> 1.780 | Grad_l2 --> 0.196 | Weights_l2 --> 11272.323 | Lr --> 0.002 | Seconds_per_step --> 4.870 |
+[2024-08-31 17:18:10,558][Main][INFO] - [train] Step 15625 out of 20000 | Loss --> 1.772 | Grad_l2 --> 0.200 | Weights_l2 --> 11272.311 | Lr --> 0.002 | Seconds_per_step --> 4.933 |
+[2024-08-31 17:20:12,172][Main][INFO] - [train] Step 15650 out of 20000 | Loss --> 1.780 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.303 | Lr --> 0.002 | Seconds_per_step --> 4.864 |
+[2024-08-31 17:22:13,878][Main][INFO] - [train] Step 15675 out of 20000 | Loss --> 1.767 | Grad_l2 --> 0.196 | Weights_l2 --> 11272.290 | Lr --> 0.002 | Seconds_per_step --> 4.868 |
+[2024-08-31 17:24:17,279][Main][INFO] - [train] Step 15700 out of 20000 | Loss --> 1.773 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.273 | Lr --> 0.002 | Seconds_per_step --> 4.936 |
+[2024-08-31 17:26:19,126][Main][INFO] - [train] Step 15725 out of 20000 | Loss --> 1.780 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.263 | Lr --> 0.002 | Seconds_per_step --> 4.874 |
+[2024-08-31 17:28:20,908][Main][INFO] - [train] Step 15750 out of 20000 | Loss --> 1.782 | Grad_l2 --> 0.200 | Weights_l2 --> 11272.246 | Lr --> 0.002 | Seconds_per_step --> 4.871 |
+[2024-08-31 17:30:24,238][Main][INFO] - [train] Step 15775 out of 20000 | Loss --> 1.768 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.233 | Lr --> 0.002 | Seconds_per_step --> 4.933 |
+[2024-08-31 17:32:26,169][Main][INFO] - [train] Step 15800 out of 20000 | Loss --> 1.760 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.220 | Lr --> 0.002 | Seconds_per_step --> 4.877 |
+[2024-08-31 17:34:28,677][Main][INFO] - [train] Step 15825 out of 20000 | Loss --> 1.766 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.207 | Lr --> 0.002 | Seconds_per_step --> 4.900 |
+[2024-08-31 17:36:32,092][Main][INFO] - [train] Step 15850 out of 20000 | Loss --> 1.781 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.196 | Lr --> 0.002 | Seconds_per_step --> 4.936 |
+[2024-08-31 17:38:33,746][Main][INFO] - [train] Step 15875 out of 20000 | Loss --> 1.763 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.177 | Lr --> 0.002 | Seconds_per_step --> 4.866 |
+[2024-08-31 17:40:35,415][Main][INFO] - [train] Step 15900 out of 20000 | Loss --> 1.770 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.165 | Lr --> 0.002 | Seconds_per_step --> 4.867 |
+[2024-08-31 17:42:36,852][Main][INFO] - [train] Step 15925 out of 20000 | Loss --> 1.766 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.154 | Lr --> 0.002 | Seconds_per_step --> 4.857 |
+[2024-08-31 17:44:40,125][Main][INFO] - [train] Step 15950 out of 20000 | Loss --> 1.774 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.141 | Lr --> 0.002 | Seconds_per_step --> 4.931 |
+[2024-08-31 17:46:42,051][Main][INFO] - [train] Step 15975 out of 20000 | Loss --> 1.761 | Grad_l2 --> 0.196 | Weights_l2 --> 11272.121 | Lr --> 0.002 | Seconds_per_step --> 4.877 |
+[2024-08-31 17:48:43,831][Main][INFO] - [train] Step 16000 out of 20000 | Loss --> 1.775 | Grad_l2 --> 0.196 | Weights_l2 --> 11272.110 | Lr --> 0.002 | Seconds_per_step --> 4.871 |
+[2024-08-31 17:50:47,287][Main][INFO] - [train] Step 16025 out of 20000 | Loss --> 1.768 | Grad_l2 --> 0.197 | Weights_l2 --> 11272.098 | Lr --> 0.002 | Seconds_per_step --> 4.938 |
+[2024-08-31 17:52:48,992][Main][INFO] - [train] Step 16050 out of 20000 | Loss --> 1.770 | Grad_l2 --> 0.202 | Weights_l2 --> 11272.084 | Lr --> 0.002 | Seconds_per_step --> 4.868 |
+[2024-08-31 17:54:50,724][Main][INFO] - [train] Step 16075 out of 20000 | Loss --> 1.767 | Grad_l2 --> 0.199 | Weights_l2 --> 11272.068 | Lr --> 0.002 | Seconds_per_step --> 4.869 |
+[2024-08-31 17:56:54,016][Main][INFO] - [train] Step 16100 out of 20000 | Loss --> 1.777 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.054 | Lr --> 0.002 | Seconds_per_step --> 4.932 |
+[2024-08-31 17:58:55,942][Main][INFO] - [train] Step 16125 out of 20000 | Loss --> 1.768 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.037 | Lr --> 0.002 | Seconds_per_step --> 4.877 |
+[2024-08-31 18:00:57,478][Main][INFO] - [train] Step 16150 out of 20000 | Loss --> 1.766 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.020 | Lr --> 0.002 | Seconds_per_step --> 4.861 |
+[2024-08-31 18:03:00,736][Main][INFO] - [train] Step 16175 out of 20000 | Loss --> 1.767 | Grad_l2 --> 0.198 | Weights_l2 --> 11272.000 | Lr --> 0.002 | Seconds_per_step --> 4.930 |
+[2024-08-31 18:05:02,586][Main][INFO] - [train] Step 16200 out of 20000 | Loss --> 1.760 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.988 | Lr --> 0.002 | Seconds_per_step --> 4.874 |
+[2024-08-31 18:07:04,438][Main][INFO] - [train] Step 16225 out of 20000 | Loss --> 1.758 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.975 | Lr --> 0.002 | Seconds_per_step --> 4.874 |
+[2024-08-31 18:09:07,901][Main][INFO] - [train] Step 16250 out of 20000 | Loss --> 1.763 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.962 | Lr --> 0.001 | Seconds_per_step --> 4.938 |
+[2024-08-31 18:11:09,649][Main][INFO] - [train] Step 16275 out of 20000 | Loss --> 1.749 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.952 | Lr --> 0.001 | Seconds_per_step --> 4.870 |
+[2024-08-31 18:13:11,767][Main][INFO] - [train] Step 16300 out of 20000 | Loss --> 1.754 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.935 | Lr --> 0.001 | Seconds_per_step --> 4.885 |
+[2024-08-31 18:15:13,819][Main][INFO] - [train] Step 16325 out of 20000 | Loss --> 1.765 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.928 | Lr --> 0.001 | Seconds_per_step --> 4.882 |
+[2024-08-31 18:17:17,096][Main][INFO] - [train] Step 16350 out of 20000 | Loss --> 1.749 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.916 | Lr --> 0.001 | Seconds_per_step --> 4.931 |
+[2024-08-31 18:19:18,879][Main][INFO] - [train] Step 16375 out of 20000 | Loss --> 1.757 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.902 | Lr --> 0.001 | Seconds_per_step --> 4.871 |
+[2024-08-31 18:21:20,566][Main][INFO] - [train] Step 16400 out of 20000 | Loss --> 1.770 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.886 | Lr --> 0.001 | Seconds_per_step --> 4.867 |
+[2024-08-31 18:23:24,550][Main][INFO] - [train] Step 16425 out of 20000 | Loss --> 1.752 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.874 | Lr --> 0.001 | Seconds_per_step --> 4.959 |
+[2024-08-31 18:25:26,461][Main][INFO] - [train] Step 16450 out of 20000 | Loss --> 1.756 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.860 | Lr --> 0.001 | Seconds_per_step --> 4.876 |
+[2024-08-31 18:27:28,278][Main][INFO] - [train] Step 16475 out of 20000 | Loss --> 1.754 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.846 | Lr --> 0.001 | Seconds_per_step --> 4.873 |
+[2024-08-31 18:29:32,921][Main][INFO] - [train] Step 16500 out of 20000 | Loss --> 1.753 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.834 | Lr --> 0.001 | Seconds_per_step --> 4.986 |
+[2024-08-31 18:31:35,149][Main][INFO] - [train] Step 16525 out of 20000 | Loss --> 1.760 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.819 | Lr --> 0.001 | Seconds_per_step --> 4.889 |
+[2024-08-31 18:33:37,364][Main][INFO] - [train] Step 16550 out of 20000 | Loss --> 1.754 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.799 | Lr --> 0.001 | Seconds_per_step --> 4.889 |
+[2024-08-31 18:35:40,915][Main][INFO] - [train] Step 16575 out of 20000 | Loss --> 1.751 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.786 | Lr --> 0.001 | Seconds_per_step --> 4.942 |
+[2024-08-31 18:37:43,466][Main][INFO] - [train] Step 16600 out of 20000 | Loss --> 1.768 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.770 | Lr --> 0.001 | Seconds_per_step --> 4.902 |
+[2024-08-31 18:39:45,637][Main][INFO] - [train] Step 16625 out of 20000 | Loss --> 1.749 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.754 | Lr --> 0.001 | Seconds_per_step --> 4.887 |
+[2024-08-31 18:41:49,405][Main][INFO] - [train] Step 16650 out of 20000 | Loss --> 1.748 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.739 | Lr --> 0.001 | Seconds_per_step --> 4.951 |
+[2024-08-31 18:43:51,102][Main][INFO] - [train] Step 16675 out of 20000 | Loss --> 1.745 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.723 | Lr --> 0.001 | Seconds_per_step --> 4.868 |
+[2024-08-31 18:45:52,944][Main][INFO] - [train] Step 16700 out of 20000 | Loss --> 1.736 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.708 | Lr --> 0.001 | Seconds_per_step --> 4.874 |
+[2024-08-31 18:47:56,099][Main][INFO] - [train] Step 16725 out of 20000 | Loss --> 1.757 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.698 | Lr --> 0.001 | Seconds_per_step --> 4.926 |
+[2024-08-31 18:49:57,801][Main][INFO] - [train] Step 16750 out of 20000 | Loss --> 1.742 | Grad_l2 --> 0.195 | Weights_l2 --> 11271.684 | Lr --> 0.001 | Seconds_per_step --> 4.868 |
+[2024-08-31 18:51:59,437][Main][INFO] - [train] Step 16775 out of 20000 | Loss --> 1.755 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.665 | Lr --> 0.001 | Seconds_per_step --> 4.865 |
+[2024-08-31 18:54:00,855][Main][INFO] - [train] Step 16800 out of 20000 | Loss --> 1.747 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.651 | Lr --> 0.001 | Seconds_per_step --> 4.857 |
+[2024-08-31 18:56:03,853][Main][INFO] - [train] Step 16825 out of 20000 | Loss --> 1.743 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.632 | Lr --> 0.001 | Seconds_per_step --> 4.920 |
+[2024-08-31 18:58:05,201][Main][INFO] - [train] Step 16850 out of 20000 | Loss --> 1.745 | Grad_l2 --> 0.201 | Weights_l2 --> 11271.619 | Lr --> 0.001 | Seconds_per_step --> 4.854 |
+[2024-08-31 19:00:06,563][Main][INFO] - [train] Step 16875 out of 20000 | Loss --> 1.753 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.606 | Lr --> 0.001 | Seconds_per_step --> 4.854 |
+[2024-08-31 19:02:09,317][Main][INFO] - [train] Step 16900 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.591 | Lr --> 0.001 | Seconds_per_step --> 4.910 |
+[2024-08-31 19:04:10,472][Main][INFO] - [train] Step 16925 out of 20000 | Loss --> 1.729 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.575 | Lr --> 0.001 | Seconds_per_step --> 4.846 |
+[2024-08-31 19:06:11,583][Main][INFO] - [train] Step 16950 out of 20000 | Loss --> 1.741 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.559 | Lr --> 0.001 | Seconds_per_step --> 4.844 |
+[2024-08-31 19:08:14,363][Main][INFO] - [train] Step 16975 out of 20000 | Loss --> 1.729 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.545 | Lr --> 0.001 | Seconds_per_step --> 4.911 |
+[2024-08-31 19:10:15,178][Main][INFO] - [train] Step 17000 out of 20000 | Loss --> 1.738 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.533 | Lr --> 0.001 | Seconds_per_step --> 4.832 |
+[2024-08-31 19:12:16,110][Main][INFO] - [train] Step 17025 out of 20000 | Loss --> 1.741 | Grad_l2 --> 0.200 | Weights_l2 --> 11271.515 | Lr --> 0.001 | Seconds_per_step --> 4.837 |
+[2024-08-31 19:14:18,572][Main][INFO] - [train] Step 17050 out of 20000 | Loss --> 1.740 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.500 | Lr --> 0.001 | Seconds_per_step --> 4.898 |
+[2024-08-31 19:16:19,446][Main][INFO] - [train] Step 17075 out of 20000 | Loss --> 1.735 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.485 | Lr --> 0.001 | Seconds_per_step --> 4.835 |
+[2024-08-31 19:18:20,449][Main][INFO] - [train] Step 17100 out of 20000 | Loss --> 1.734 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.471 | Lr --> 0.001 | Seconds_per_step --> 4.840 |
+[2024-08-31 19:20:23,025][Main][INFO] - [train] Step 17125 out of 20000 | Loss --> 1.736 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.453 | Lr --> 0.001 | Seconds_per_step --> 4.903 |
+[2024-08-31 19:22:23,946][Main][INFO] - [train] Step 17150 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.439 | Lr --> 0.001 | Seconds_per_step --> 4.837 |
+[2024-08-31 19:24:25,361][Main][INFO] - [train] Step 17175 out of 20000 | Loss --> 1.732 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.422 | Lr --> 0.001 | Seconds_per_step --> 4.857 |
+[2024-08-31 19:26:26,446][Main][INFO] - [train] Step 17200 out of 20000 | Loss --> 1.734 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.408 | Lr --> 0.001 | Seconds_per_step --> 4.843 |
+[2024-08-31 19:28:29,313][Main][INFO] - [train] Step 17225 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.395 | Lr --> 0.001 | Seconds_per_step --> 4.915 |
+[2024-08-31 19:30:30,859][Main][INFO] - [train] Step 17250 out of 20000 | Loss --> 1.737 | Grad_l2 --> 0.195 | Weights_l2 --> 11271.382 | Lr --> 0.001 | Seconds_per_step --> 4.862 |
+[2024-08-31 19:32:32,558][Main][INFO] - [train] Step 17275 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.367 | Lr --> 0.001 | Seconds_per_step --> 4.868 |
+[2024-08-31 19:34:35,555][Main][INFO] - [train] Step 17300 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.349 | Lr --> 0.001 | Seconds_per_step --> 4.920 |
+[2024-08-31 19:36:37,051][Main][INFO] - [train] Step 17325 out of 20000 | Loss --> 1.722 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.333 | Lr --> 0.001 | Seconds_per_step --> 4.860 |
+[2024-08-31 19:38:38,163][Main][INFO] - [train] Step 17350 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.315 | Lr --> 0.001 | Seconds_per_step --> 4.844 |
+[2024-08-31 19:40:41,121][Main][INFO] - [train] Step 17375 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.299 | Lr --> 0.001 | Seconds_per_step --> 4.918 |
+[2024-08-31 19:42:42,654][Main][INFO] - [train] Step 17400 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.201 | Weights_l2 --> 11271.281 | Lr --> 0.001 | Seconds_per_step --> 4.861 |
+[2024-08-31 19:44:44,126][Main][INFO] - [train] Step 17425 out of 20000 | Loss --> 1.713 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.264 | Lr --> 0.001 | Seconds_per_step --> 4.859 |
+[2024-08-31 19:46:47,266][Main][INFO] - [train] Step 17450 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.200 | Weights_l2 --> 11271.245 | Lr --> 0.001 | Seconds_per_step --> 4.925 |
+[2024-08-31 19:48:48,692][Main][INFO] - [train] Step 17475 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.227 | Lr --> 0.001 | Seconds_per_step --> 4.857 |
+[2024-08-31 19:50:50,523][Main][INFO] - [train] Step 17500 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.212 | Lr --> 0.001 | Seconds_per_step --> 4.873 |
+[2024-08-31 19:50:50,523][accelerate.accelerator][INFO] - Saving current state to checkpoint-pt-17500
+[2024-08-31 19:50:50,530][accelerate.utils.other][WARNING] - Removed shared tensor {'decoder.embed_tokens.weight', 'encoder.embed_tokens.weight'} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading
+[2024-08-31 19:50:57,172][accelerate.checkpointing][INFO] - Model weights saved in checkpoint-pt-17500/model.safetensors
+[2024-08-31 19:51:06,254][accelerate.checkpointing][INFO] - Optimizer state saved in checkpoint-pt-17500/optimizer.bin
+[2024-08-31 19:51:06,257][accelerate.checkpointing][INFO] - Scheduler state saved in checkpoint-pt-17500/scheduler.bin
+[2024-08-31 19:51:06,258][accelerate.checkpointing][INFO] - Sampler state for dataloader 0 saved in checkpoint-pt-17500/sampler.bin
+[2024-08-31 19:51:06,260][accelerate.checkpointing][INFO] - Sampler state for dataloader 1 saved in checkpoint-pt-17500/sampler_1.bin
+[2024-08-31 19:51:06,261][accelerate.checkpointing][INFO] - Random states saved in checkpoint-pt-17500/random_states_0.pkl
+[2024-08-31 19:53:08,757][Main][INFO] - [train] Step 17525 out of 20000 | Loss --> 1.716 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.193 | Lr --> 0.001 | Seconds_per_step --> 5.529 |
+[2024-08-31 19:55:09,923][Main][INFO] - [train] Step 17550 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.173 | Lr --> 0.001 | Seconds_per_step --> 4.847 |
+[2024-08-31 19:57:11,243][Main][INFO] - [train] Step 17575 out of 20000 | Loss --> 1.700 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.155 | Lr --> 0.001 | Seconds_per_step --> 4.853 |
+[2024-08-31 19:59:14,099][Main][INFO] - [train] Step 17600 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.137 | Lr --> 0.001 | Seconds_per_step --> 4.914 |
+[2024-08-31 20:01:15,562][Main][INFO] - [train] Step 17625 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.197 | Weights_l2 --> 11271.118 | Lr --> 0.001 | Seconds_per_step --> 4.858 |
+[2024-08-31 20:03:16,470][Main][INFO] - [train] Step 17650 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.199 | Weights_l2 --> 11271.097 | Lr --> 0.001 | Seconds_per_step --> 4.836 |
+[2024-08-31 20:05:17,916][Main][INFO] - [train] Step 17675 out of 20000 | Loss --> 1.733 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.078 | Lr --> 0.001 | Seconds_per_step --> 4.858 |
+[2024-08-31 20:07:20,683][Main][INFO] - [train] Step 17700 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.196 | Weights_l2 --> 11271.062 | Lr --> 0.001 | Seconds_per_step --> 4.911 |
+[2024-08-31 20:09:22,414][Main][INFO] - [train] Step 17725 out of 20000 | Loss --> 1.707 | Grad_l2 --> 0.195 | Weights_l2 --> 11271.041 | Lr --> 0.001 | Seconds_per_step --> 4.869 |
+[2024-08-31 20:11:24,033][Main][INFO] - [train] Step 17750 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.024 | Lr --> 0.001 | Seconds_per_step --> 4.865 |
+[2024-08-31 20:13:26,602][Main][INFO] - [train] Step 17775 out of 20000 | Loss --> 1.730 | Grad_l2 --> 0.198 | Weights_l2 --> 11271.007 | Lr --> 0.001 | Seconds_per_step --> 4.903 |
+[2024-08-31 20:15:27,607][Main][INFO] - [train] Step 17800 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.991 | Lr --> 0.001 | Seconds_per_step --> 4.840 |
+[2024-08-31 20:17:28,616][Main][INFO] - [train] Step 17825 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.976 | Lr --> 0.001 | Seconds_per_step --> 4.840 |
+[2024-08-31 20:19:31,033][Main][INFO] - [train] Step 17850 out of 20000 | Loss --> 1.729 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.961 | Lr --> 0.001 | Seconds_per_step --> 4.897 |
+[2024-08-31 20:21:32,133][Main][INFO] - [train] Step 17875 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.946 | Lr --> 0.001 | Seconds_per_step --> 4.844 |
+[2024-08-31 20:23:33,151][Main][INFO] - [train] Step 17900 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.933 | Lr --> 0.000 | Seconds_per_step --> 4.841 |
+[2024-08-31 20:25:35,828][Main][INFO] - [train] Step 17925 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.921 | Lr --> 0.000 | Seconds_per_step --> 4.907 |
+[2024-08-31 20:27:36,892][Main][INFO] - [train] Step 17950 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.911 | Lr --> 0.000 | Seconds_per_step --> 4.842 |
+[2024-08-31 20:29:38,066][Main][INFO] - [train] Step 17975 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.900 | Lr --> 0.000 | Seconds_per_step --> 4.847 |
+[2024-08-31 20:31:40,569][Main][INFO] - [train] Step 18000 out of 20000 | Loss --> 1.738 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.891 | Lr --> 0.000 | Seconds_per_step --> 4.900 |
+[2024-08-31 20:33:41,408][Main][INFO] - [train] Step 18025 out of 20000 | Loss --> 1.739 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.884 | Lr --> 0.000 | Seconds_per_step --> 4.833 |
+[2024-08-31 20:35:42,352][Main][INFO] - [train] Step 18050 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.876 | Lr --> 0.000 | Seconds_per_step --> 4.838 |
+[2024-08-31 20:37:45,322][Main][INFO] - [train] Step 18075 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.869 | Lr --> 0.000 | Seconds_per_step --> 4.919 |
+[2024-08-31 20:39:46,981][Main][INFO] - [train] Step 18100 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.862 | Lr --> 0.000 | Seconds_per_step --> 4.866 |
+[2024-08-31 20:41:48,584][Main][INFO] - [train] Step 18125 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.855 | Lr --> 0.000 | Seconds_per_step --> 4.864 |
+[2024-08-31 20:43:49,907][Main][INFO] - [train] Step 18150 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.853 |
+[2024-08-31 20:45:52,968][Main][INFO] - [train] Step 18175 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.844 | Lr --> 0.000 | Seconds_per_step --> 4.922 |
+[2024-08-31 20:47:54,325][Main][INFO] - [train] Step 18200 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.854 |
+[2024-08-31 20:49:55,663][Main][INFO] - [train] Step 18225 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.195 | Weights_l2 --> 11270.840 | Lr --> 0.000 | Seconds_per_step --> 4.853 |
+[2024-08-31 20:51:58,657][Main][INFO] - [train] Step 18250 out of 20000 | Loss --> 1.718 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.836 | Lr --> 0.000 | Seconds_per_step --> 4.920 |
+[2024-08-31 20:54:00,083][Main][INFO] - [train] Step 18275 out of 20000 | Loss --> 1.721 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.834 | Lr --> 0.000 | Seconds_per_step --> 4.857 |
+[2024-08-31 20:56:01,850][Main][INFO] - [train] Step 18300 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.831 | Lr --> 0.000 | Seconds_per_step --> 4.871 |
+[2024-08-31 20:58:04,690][Main][INFO] - [train] Step 18325 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.832 | Lr --> 0.000 | Seconds_per_step --> 4.913 |
+[2024-08-31 21:00:06,226][Main][INFO] - [train] Step 18350 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.832 | Lr --> 0.000 | Seconds_per_step --> 4.861 |
+[2024-08-31 21:02:07,970][Main][INFO] - [train] Step 18375 out of 20000 | Loss --> 1.728 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.831 | Lr --> 0.000 | Seconds_per_step --> 4.870 |
+[2024-08-31 21:04:11,035][Main][INFO] - [train] Step 18400 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.831 | Lr --> 0.000 | Seconds_per_step --> 4.922 |
+[2024-08-31 21:06:12,731][Main][INFO] - [train] Step 18425 out of 20000 | Loss --> 1.714 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.834 | Lr --> 0.000 | Seconds_per_step --> 4.868 |
+[2024-08-31 21:08:14,292][Main][INFO] - [train] Step 18450 out of 20000 | Loss --> 1.721 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.836 | Lr --> 0.000 | Seconds_per_step --> 4.862 |
+[2024-08-31 21:10:17,481][Main][INFO] - [train] Step 18475 out of 20000 | Loss --> 1.714 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.837 | Lr --> 0.000 | Seconds_per_step --> 4.927 |
+[2024-08-31 21:12:19,115][Main][INFO] - [train] Step 18500 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.838 | Lr --> 0.000 | Seconds_per_step --> 4.865 |
+[2024-08-31 21:14:20,604][Main][INFO] - [train] Step 18525 out of 20000 | Loss --> 1.714 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.839 | Lr --> 0.000 | Seconds_per_step --> 4.859 |
+[2024-08-31 21:16:24,832][Main][INFO] - [train] Step 18550 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.969 |
+[2024-08-31 21:18:26,217][Main][INFO] - [train] Step 18575 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.855 |
+[2024-08-31 21:20:27,684][Main][INFO] - [train] Step 18600 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.840 | Lr --> 0.000 | Seconds_per_step --> 4.859 |
+[2024-08-31 21:22:29,530][Main][INFO] - [train] Step 18625 out of 20000 | Loss --> 1.730 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.874 |
+[2024-08-31 21:24:33,115][Main][INFO] - [train] Step 18650 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.195 | Weights_l2 --> 11270.841 | Lr --> 0.000 | Seconds_per_step --> 4.943 |
+[2024-08-31 21:26:35,121][Main][INFO] - [train] Step 18675 out of 20000 | Loss --> 1.732 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.842 | Lr --> 0.000 | Seconds_per_step --> 4.880 |
+[2024-08-31 21:28:37,128][Main][INFO] - [train] Step 18700 out of 20000 | Loss --> 1.722 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.843 | Lr --> 0.000 | Seconds_per_step --> 4.880 |
+[2024-08-31 21:30:40,396][Main][INFO] - [train] Step 18725 out of 20000 | Loss --> 1.712 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.843 | Lr --> 0.000 | Seconds_per_step --> 4.931 |
+[2024-08-31 21:33:03,172][Main][INFO] - [train] Step 18750 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.844 | Lr --> 0.000 | Seconds_per_step --> 5.711 |
+[2024-08-31 21:35:04,721][Main][INFO] - [train] Step 18775 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.845 | Lr --> 0.000 | Seconds_per_step --> 4.862 |
+[2024-08-31 21:37:08,128][Main][INFO] - [train] Step 18800 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.845 | Lr --> 0.000 | Seconds_per_step --> 4.936 |
+[2024-08-31 21:39:09,857][Main][INFO] - [train] Step 18825 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.201 | Weights_l2 --> 11270.845 | Lr --> 0.000 | Seconds_per_step --> 4.869 |
+[2024-08-31 21:41:11,700][Main][INFO] - [train] Step 18850 out of 20000 | Loss --> 1.725 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.847 | Lr --> 0.000 | Seconds_per_step --> 4.874 |
+[2024-08-31 21:43:15,117][Main][INFO] - [train] Step 18875 out of 20000 | Loss --> 1.722 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.846 | Lr --> 0.000 | Seconds_per_step --> 4.937 |
+[2024-08-31 21:45:19,433][Main][INFO] - [train] Step 18900 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.847 | Lr --> 0.000 | Seconds_per_step --> 4.973 |
+[2024-08-31 21:47:29,032][Main][INFO] - [train] Step 18925 out of 20000 | Loss --> 1.709 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.847 | Lr --> 0.000 | Seconds_per_step --> 5.184 |
+[2024-08-31 21:49:33,512][Main][INFO] - [train] Step 18950 out of 20000 | Loss --> 1.731 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.848 | Lr --> 0.000 | Seconds_per_step --> 4.979 |
+[2024-08-31 21:51:35,196][Main][INFO] - [train] Step 18975 out of 20000 | Loss --> 1.721 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.867 |
+[2024-08-31 21:53:36,788][Main][INFO] - [train] Step 19000 out of 20000 | Loss --> 1.718 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.864 |
+[2024-08-31 21:55:38,313][Main][INFO] - [train] Step 19025 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.861 |
+[2024-08-31 21:57:41,329][Main][INFO] - [train] Step 19050 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.921 |
+[2024-08-31 21:59:42,853][Main][INFO] - [train] Step 19075 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.861 |
+[2024-08-31 22:01:44,492][Main][INFO] - [train] Step 19100 out of 20000 | Loss --> 1.730 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.865 |
+[2024-08-31 22:03:47,660][Main][INFO] - [train] Step 19125 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.927 |
+[2024-08-31 22:05:49,133][Main][INFO] - [train] Step 19150 out of 20000 | Loss --> 1.712 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.859 |
+[2024-08-31 22:07:50,623][Main][INFO] - [train] Step 19175 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.849 | Lr --> 0.000 | Seconds_per_step --> 4.860 |
+[2024-08-31 22:09:53,873][Main][INFO] - [train] Step 19200 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.930 |
+[2024-08-31 22:11:55,529][Main][INFO] - [train] Step 19225 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.866 |
+[2024-08-31 22:13:57,272][Main][INFO] - [train] Step 19250 out of 20000 | Loss --> 1.730 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.870 |
+[2024-08-31 22:16:01,229][Main][INFO] - [train] Step 19275 out of 20000 | Loss --> 1.722 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.958 |
+[2024-08-31 22:18:03,766][Main][INFO] - [train] Step 19300 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.901 |
+[2024-08-31 22:20:06,053][Main][INFO] - [train] Step 19325 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.891 |
+[2024-08-31 22:22:09,832][Main][INFO] - [train] Step 19350 out of 20000 | Loss --> 1.726 | Grad_l2 --> 0.200 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.951 |
+[2024-08-31 22:24:11,539][Main][INFO] - [train] Step 19375 out of 20000 | Loss --> 1.707 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.850 | Lr --> 0.000 | Seconds_per_step --> 4.868 |
+[2024-08-31 22:26:13,254][Main][INFO] - [train] Step 19400 out of 20000 | Loss --> 1.718 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.869 |
+[2024-08-31 22:28:15,089][Main][INFO] - [train] Step 19425 out of 20000 | Loss --> 1.711 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.851 | Lr --> 0.000 | Seconds_per_step --> 4.873 |
+[2024-08-31 22:30:18,617][Main][INFO] - [train] Step 19450 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.941 |
+[2024-08-31 22:32:20,460][Main][INFO] - [train] Step 19475 out of 20000 | Loss --> 1.711 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.874 |
+[2024-08-31 22:34:21,991][Main][INFO] - [train] Step 19500 out of 20000 | Loss --> 1.723 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.861 |
+[2024-08-31 22:36:24,959][Main][INFO] - [train] Step 19525 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.919 |
+[2024-08-31 22:38:26,479][Main][INFO] - [train] Step 19550 out of 20000 | Loss --> 1.727 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.861 |
+[2024-08-31 22:40:28,432][Main][INFO] - [train] Step 19575 out of 20000 | Loss --> 1.714 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.878 |
+[2024-08-31 22:42:31,962][Main][INFO] - [train] Step 19600 out of 20000 | Loss --> 1.718 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.941 |
+[2024-08-31 22:44:33,970][Main][INFO] - [train] Step 19625 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.880 |
+[2024-08-31 22:46:38,990][Main][INFO] - [train] Step 19650 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 5.001 |
+[2024-08-31 22:48:42,541][Main][INFO] - [train] Step 19675 out of 20000 | Loss --> 1.711 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.942 |
+[2024-08-31 22:50:48,499][Main][INFO] - [train] Step 19700 out of 20000 | Loss --> 1.724 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 5.038 |
+[2024-08-31 22:52:50,152][Main][INFO] - [train] Step 19725 out of 20000 | Loss --> 1.717 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.866 |
+[2024-08-31 22:54:53,252][Main][INFO] - [train] Step 19750 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.924 |
+[2024-08-31 22:56:55,118][Main][INFO] - [train] Step 19775 out of 20000 | Loss --> 1.712 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.875 |
+[2024-08-31 22:58:56,863][Main][INFO] - [train] Step 19800 out of 20000 | Loss --> 1.715 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.870 |
+[2024-08-31 23:01:01,950][Main][INFO] - [train] Step 19825 out of 20000 | Loss --> 1.710 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 5.003 |
+[2024-08-31 23:03:04,913][Main][INFO] - [train] Step 19850 out of 20000 | Loss --> 1.713 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.852 | Lr --> 0.000 | Seconds_per_step --> 4.918 |
+[2024-08-31 23:05:06,946][Main][INFO] - [train] Step 19875 out of 20000 | Loss --> 1.710 | Grad_l2 --> 0.196 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.881 |
+[2024-08-31 23:07:08,902][Main][INFO] - [train] Step 19900 out of 20000 | Loss --> 1.711 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.878 |
+[2024-08-31 23:09:12,065][Main][INFO] - [train] Step 19925 out of 20000 | Loss --> 1.716 | Grad_l2 --> 0.197 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.926 |
+[2024-08-31 23:11:13,586][Main][INFO] - [train] Step 19950 out of 20000 | Loss --> 1.719 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.861 |
+[2024-08-31 23:13:15,435][Main][INFO] - [train] Step 19975 out of 20000 | Loss --> 1.720 | Grad_l2 --> 0.198 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.874 |
+[2024-08-31 23:15:18,590][Main][INFO] - [train] Step 20000 out of 20000 | Loss --> 1.716 | Grad_l2 --> 0.199 | Weights_l2 --> 11270.853 | Lr --> 0.000 | Seconds_per_step --> 4.926 |
+[2024-08-31 23:15:18,591][accelerate.accelerator][INFO] - Saving current state to checkpoint-pt-20000
+[2024-08-31 23:15:18,599][accelerate.utils.other][WARNING] - Removed shared tensor {'decoder.embed_tokens.weight', 'encoder.embed_tokens.weight'} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading
+[2024-08-31 23:15:26,324][accelerate.checkpointing][INFO] - Model weights saved in checkpoint-pt-20000/model.safetensors
+[2024-08-31 23:15:35,439][accelerate.checkpointing][INFO] - Optimizer state saved in checkpoint-pt-20000/optimizer.bin
+[2024-08-31 23:15:35,440][accelerate.checkpointing][INFO] - Scheduler state saved in checkpoint-pt-20000/scheduler.bin
+[2024-08-31 23:15:35,441][accelerate.checkpointing][INFO] - Sampler state for dataloader 0 saved in checkpoint-pt-20000/sampler.bin
+[2024-08-31 23:15:35,441][accelerate.checkpointing][INFO] - Sampler state for dataloader 1 saved in checkpoint-pt-20000/sampler_1.bin
+[2024-08-31 23:15:35,442][accelerate.checkpointing][INFO] - Random states saved in checkpoint-pt-20000/random_states_0.pkl
+[2024-08-31 23:31:42,282][Main][INFO] - [eval] Step 20001 out of 20000 | Loss --> 2.073 | Accuracy --> 0.604 | Time --> 964.275 |
+[2024-08-31 23:31:42,287][accelerate.accelerator][INFO] - Saving current state to checkpoint-pt-20001
+[2024-08-31 23:31:42,295][accelerate.utils.other][WARNING] - Removed shared tensor {'decoder.embed_tokens.weight', 'encoder.embed_tokens.weight'} while saving. This should be OK, but check by verifying that you don't receive any warning while reloading
+[2024-08-31 23:31:50,975][accelerate.checkpointing][INFO] - Model weights saved in checkpoint-pt-20001/model.safetensors
+[2024-08-31 23:32:00,717][accelerate.checkpointing][INFO] - Optimizer state saved in checkpoint-pt-20001/optimizer.bin
+[2024-08-31 23:32:00,719][accelerate.checkpointing][INFO] - Scheduler state saved in checkpoint-pt-20001/scheduler.bin
+[2024-08-31 23:32:00,720][accelerate.checkpointing][INFO] - Sampler state for dataloader 0 saved in checkpoint-pt-20001/sampler.bin
+[2024-08-31 23:32:00,720][accelerate.checkpointing][INFO] - Sampler state for dataloader 1 saved in checkpoint-pt-20001/sampler_1.bin
+[2024-08-31 23:32:00,721][accelerate.checkpointing][INFO] - Random states saved in checkpoint-pt-20001/random_states_0.pkl
+tokenizer config file saved in ./tokenizer/tokenizer_config.json
+Special tokens file saved in ./tokenizer/special_tokens_map.json

checkpoints/wandb/run-20240830_195924-mao0tqjy/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.777546563744545, "train/grad_l2": 0.1973351389169693, "train/weights_l2": 11272.363778775605, "train/lr": 0.0020558542377918645, "train/seconds_per_step": 4.877207107543946, "_timestamp": 1725124200.3571296, "_runtime": 76236.1699206829, "_step": 15525}
+{"train/loss": 1.7155881041288377, "train/grad_l2": 0.19877906143665314, "train/weights_l2": 11270.853009077697, "train/lr": 2e-05, "train/seconds_per_step": 4.926107158660889, "_timestamp": 1725147102.2811465, "_runtime": 99138.09393763542, "_step": 20001, "_wandb": {"runtime": 99156}, "eval/loss": 2.0733377319718933, "eval/accuracy": 0.6044892090926607, "eval/time": 964.2750298976898}

checkpoints/wandb/run-20240830_195924-mao0tqjy/logs/debug.log

@@ -25,3 +25,12 @@ config: {'mode': 'pt', 'device': 'gpu', 'precision': 'bf16', 'eval_only': False,
 2024-08-30 19:59:24,815 INFO    MainThread:29052 [wandb_run.py:_redirect():2399] Redirects installed.
 2024-08-30 19:59:24,818 INFO    MainThread:29052 [wandb_init.py:init():894] run started, returning control to user process
 2024-08-30 19:59:44,796 INFO    MainThread:29052 [wandb_run.py:_config_callback():1392] config_cb None None {'mode': 'pt', 'device': 'gpu', 'precision': 'bf16', 'eval_only': False, 'predict_only': False, 'seed': 34534, 'model': {'klass': 'hf_t5', 'name': 'pszemraj/tFINE-900m-e16-d32', 'overwrite': {'dropout_rate': 0.0}, 'checkpoint_path': '', 'random_init': False, 'compile': True}, 'tokenizer': {'name': 'BEE-spoke-data/slimpajama_tok-48128-BPE-forT5'}, 'data': {'input_length': 1024, 'mlm_probability': 0.15, 'mean_noise_span_length': 3.0, 'num_workers': 16, 'before_mask_input_length': 1137, 'target_length': 229}, 'optim': {'name': 'adamwscale', 'base_lr': 0.01, 'batch_size': 128, 'total_steps': 20000, 'epochs': -1, 'warmup_steps': 5000, 'lr_scheduler': 'cosine', 'weight_decay': 0.0001, 'grad_clip': 1.0, 'grad_acc': 8, 'final_cosine': 2e-05}, 'eval': {'every_steps': 1000000000, 'steps': 500, 'corrected_steps': 500}, 'checkpoint': {'every_steps': 2500}, 'logging': {'use_wandb': True, 'wandb_config': {'project': 'nanoT5', 'entity': 'pszemraj', 'tags': ['900m', '1024'], 'mode': 'online'}, 'every_steps': 25, 'grad_l2': True, 'weights_l2': True}, 'slurm_id': 'none', 'working_dir': '/workspace/nanoT5/outputs/2024-08-30/19-59-22', 'n_all_param': 887492096}
+2024-08-31 23:32:00,793 INFO    MainThread:29052 [wandb_run.py:_finish():2160] finishing run pszemraj/nanoT5/mao0tqjy
+2024-08-31 23:32:00,796 INFO    MainThread:29052 [wandb_run.py:_atexit_cleanup():2424] got exitcode: 0
+2024-08-31 23:32:00,797 INFO    MainThread:29052 [wandb_run.py:_restore():2406] restore
+2024-08-31 23:32:00,798 INFO    MainThread:29052 [wandb_run.py:_restore():2412] restore done
+2024-08-31 23:32:00,799 INFO    MainThread:29052 [wandb_run.py:_on_finish():2677] communicating current version
+2024-08-31 23:32:00,827 INFO    MainThread:29052 [wandb_run.py:_on_finish():2686] got version response 
+2024-08-31 23:32:06,426 INFO    MainThread:29052 [wandb_run.py:_footer_history_summary_info():4078] rendering history
+2024-08-31 23:32:06,427 INFO    MainThread:29052 [wandb_run.py:_footer_history_summary_info():4110] rendering summary
+2024-08-31 23:32:06,433 INFO    MainThread:29052 [wandb_run.py:_footer_sync_info():4037] logging synced files

checkpoints/wandb/run-20240830_195924-mao0tqjy/run-mao0tqjy.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34c4d82ee5fa3daa21587d65efb7972e3a2447cce764ad1cd0eaec8aa61ffb19
-size 9030581
+oid sha256:409f487e6f685193cdefb06459ee5126b8198454666c19a7278051ce8b773999
+size 11743810